{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Double layer NN for XOR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "tf.set_random_seed(777)  # for reproducibility"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NN 2 -> 2 -> 1 fitting just 4 data points (4 quadrants)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# input & output\n",
    "X = tf.placeholder(tf.float32, [None, 2])\n",
    "Y = tf.placeholder(tf.float32, [None, 1])\n",
    "\n",
    "# 1st layer\n",
    "W1 = tf.Variable(tf.random_normal([2, 2]), name='weight1')\n",
    "b1 = tf.Variable(tf.random_normal([2]), name='bias1')\n",
    "layer1 = tf.sigmoid(tf.matmul(X, W1) + b1)\n",
    "\n",
    "# 2nd layer\n",
    "W2 = tf.Variable(tf.random_normal([2, 1]), name='weight2')\n",
    "b2 = tf.Variable(tf.random_normal([1]), name='bias2')\n",
    "hypothesis = tf.sigmoid(tf.matmul(layer1, W2) + b2)\n",
    "\n",
    "# cost/loss function\n",
    "cost = -tf.reduce_mean(Y*tf.log(hypothesis) + (1 - Y)*tf.log(1 - hypothesis))\n",
    "train = tf.train.GradientDescentOptimizer(learning_rate=0.1).minimize(cost)\n",
    "\n",
    "# Accuracy computation\n",
    "# True if hypothesis>0.5 else False\n",
    "predicted = tf.cast(hypothesis > 0.5, dtype=tf.float32)\n",
    "accuracy = tf.reduce_mean(tf.cast(tf.equal(predicted, Y), dtype=tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_data = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], dtype=np.float32)\n",
    "y_data = np.array([[0], [1], [1], [0]], dtype=np.float32)\n",
    "feed_dict={X: x_data, Y: y_data}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.7379836\n",
      "4000 0.38647872\n",
      "8000 0.027617319\n",
      "12000 0.013000652\n",
      "16000 0.008423076\n",
      "20000 0.006210422\n",
      "\n",
      "Hypothesis:  [[0.00600311]\n",
      " [0.99184275]\n",
      " [0.99452436]\n",
      " [0.00512595]] \n",
      "Correct:  [[0.]\n",
      " [1.]\n",
      " [1.]\n",
      " [0.]] \n",
      "Accuracy:  1.0\n"
     ]
    }
   ],
   "source": [
    "# Launch graph\n",
    "with tf.Session() as sess:\n",
    "    # Initialize TensorFlow variables\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    for step in range(20001):\n",
    "        sess.run(train, feed_dict={X: x_data, Y: y_data})\n",
    "        if step % 4000 == 0:\n",
    "            print(step, sess.run(cost, feed_dict=feed_dict))\n",
    "\n",
    "    # Accuracy report\n",
    "    h, c, a = sess.run([hypothesis, predicted, accuracy], feed_dict=feed_dict)\n",
    "    print(\"\\nHypothesis: \", h, \"\\nCorrect: \", c, \"\\nAccuracy: \", a)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XOR dataset from Python Machine Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X9sXtd5H/DvQ0+NCpeUbEeOnDCAPbRI1hkJAzFihQ32tnqrERRxW6BAjWFr4AHCgEx8SbVAmxmQqC3BNiSlSHoBVgMO1gKpiwKZs6Br0yTYki6oxZQK6Myp7cALMIRLnaiqJdlY3dp6n/1xefXe9/L+Pufce8693w/wgrovX9573lfSfXjOec5zRFVBRETkm5muG0BERJSFAYqIiLzEAEVERF5igCIiIi8xQBERkZcYoIiIyEvGAUpEDovIN0TkORH5tohcsNEwIiIaNjFdByUiAuB2VX1dRA4B+DqAkapestFAIiIapr9legKNItzr+4eH9h9c/UtEREaMAxQAiMhtAC4D+HEAn1bV7YzXnAZwGgBuv/32E+9973ttXJqIiAJz+fLlv1DVY2WvMx7imzqZyFEAzwA4o6rP571ucXFRd3Z2rF2XiIjCISKXVXWx7HVWs/hU9RqArwJ42OZ5iYhoeGxk8R3b7zlBRH4UwEMAXjQ9LxERDZuNOah7APzW/jzUDIDfU9Xft3BeIiIaMBtZfN8C8AELbSEiohxvvvkm9vb28MYbb3TdlMoOHz6M+fl5HDp0qNHPW8niIyIit/b29jA7O4t7770X0fJTv6kqrl69ir29Pdx3332NzsFSR0REAXjjjTdw1113BRGcAEBEcNdddxn1+BigiIgCEUpwipm2lwGKiIi8xABFRESVPPbYY7j77rtx//33t3I9Bigioh5KFwmyUTToIx/5CL74xS+an6giBigiop5ZWwNWVydBSTU6XlszO+8DDzyAO++807R5lTFAERH1iCpw7RqwuTkJUqur0fG1a3Z6Um3hOigioh4RAS5ejP68uRk9AGA0ip4PKRGQPSgiop5JBqlYaMEJYIAiIuqdeFgvKTknFQoGKCKiHknOOY1GwHgcfU3OSTX16KOP4tSpU3jppZcwPz+Pp556yl7DM3AOioioR0SAo0en55zi4b6jR82G+Z5++mk7jayIAYqIqGfW1qKeUhyM4iDFOSgiIupcOhiFFpwABigiIvIUAxQREXmJAYqIiLzEAEVERF5igCIiotpefPFFnDp1Cm9729vwqU99ysk1mGZORES13Xnnndja2sLnP/95Z9dgD4rIkIt9d4iMzM1FeeXpx9yctUvcfffd+OAHP4hDhw5ZO2caAxSRAVf77vRKCzdLSnnttXrPe4oBiqihPu2741RPbpbUPgYooobi8jFxIc6ZmUmBzhDLyhCV+fSnP42FhQUsLCzg+9//vvPrMUARGejLvjtEVXz0ox/F7u4udnd38c53vtP59RigiAz0Zd8dorpeeeUVzM/PY319HR//+McxPz+PGzduWL0GAxRRQy733SEyMjtb7/kGjh8/jr29Pdy4cQPXrl3D3t4e5iwnvnAdFFFDLvfd6ZXZ2eyECIs3S0qx3JPpCgNUC5L7smQdU7j6su+OUz25WVL7OMTnGNfJ9F8f9t2hMGhg48am7WWAcojrZIjIlsOHD+Pq1avBBClVxdWrV3H48OHG5zAe4hORdwP4bQDHAYwBPKmqm6bn7YPknMTmZvQAuE6GiOqbn5/H3t4erly50nVTKjt8+DDm5+cb/7yYRmMRuQfAPar6TRGZBXAZwM+p6p/l/czi4qLu7OwYXTckqtEizth4bB6cOK9FRKESkcuqulj2OuMhPlX9c1X95v6fXwPwAoB3mZ63L1ysk/FpXiu0QqmhtZdoyKzOQYnIvQA+AGA743unRWRHRHZC6qKacLFOxqd5LReB0mUA8SmwE1E5awFKRH4MwOcArKjqgbxSVX1SVRdVdfHYsWO2Luu1vHUyo1HzdTK+1H9zEShdBhCfAjsRVaSqxg8AhwD8EYCzVV5/4sQJHZLxuPi46Tmj22r0sHHOJm0YjabbMRo1a0vyXPE50scu2ru8PH3uLj5HasHs7PRffPyYne26ZYMEYEcrxArjHpSICICnALygquum5+sj2+tkfKn/ZrNQahs9w6z2JnHIr8e45UeQbAzx/T0A/wzAPxKR3f3HhyyclzL4VP/NdqB0XRk8q71bW8DKCof8iHxkI4vv66oqqvo+VV3Yf/yBjcbRQS7mtZpwlQDiqmeY1d7l5eh7W1vcy4nIR6zFFyAf6r/ZLpSaDiAXL06OAfP3l9XejY3oe1tbk9c5/Rzn5vKLprJeHdEBDFCB8qH+m81A2UZl8HR7s6yuOgxSnAchqoUBiozYDJRt9Azjc8U9tq0tNz22+BocKvQEt/wIEgMUeaWtnqHrHtvaWpRsEZ9bFWCs6hCHUIPEAEWFfK/5Z9I+Vz225KJgYNI72zA7LdHgcLsNyuV7aSAb7XPRY8tb00VE9TBAUSbfSwP53r6sNV2aN9/BeRCiTBzio0y+72Xle/sy13Q9dsOLthGFwng/qCaGth9UyFzsZWWTj+0rWtPlSwAl6lJr+0FRf/lS8y8pee3xOCpTlNR1+wB/qn0QhY5DfJTJdWWHJpKp22trwBe+AOzuAktLwJ/8CXDiRLftS7e162ofRKFjgKJMbVR2qCOZFBH3kHZ3o68nT0bBc3cXWFgAjhzxIxD4UO2DKGScg6JCPq2DSvbqsoxGwPr69JwUEfmHc1BkhU+9gLL9nC5eZHCiCubmon9M6cfcXFjXGAD+d6ZgZCVtJPmQIGGMNzb32ijay8LAVnAOioKQHN5L7uMETI59SZAwwhsb0S0MUBSEdNLGhQuTwHTHHcD585PXBBuciGgKkyQoKMkkjfifbvI4+OBU9AaCH7/0RBufMf8eCzFJgnop+f8+np7J+h5RsDgPeQsDFFGIeBNrro2ivSbX4DzkLZyDIvJJ1Z1f27iJzc3ltyXkDQDbaHvIn49HGKCIfOLTjY2/yVPHOMRHhIPz1pzHJuoeAxQ5FcKN3/edg4mGigGKnAnhxu/7zryE4SWEcOflWzgHRU4kb/zAwU37fFmz5PvOvLmqJlP0wdDmwnyah+wYF+qSM1nVx7Nu/D5UTLexM68P78OqOll8LjP+uOi1d7hQlzqXVX08HZx8GAa0sXOwD+/Duhs3ojeSfmQFnKH1cqgVDFDkTNmN34f5n/TOweNx9DXZpirn6Pp9EPUR56DICVVgZSWqOB4P662sHKw43vX8j42dg0WiXXwXFqbfh0+7+wYjb6iQBokBipy4cAHY3o4qjieH+ZaWpm/8cUBIzlO1nZywtjY9XxS3qWobVIHr1ydb0Md2d4EHHujBXFSb6gSnPiaE0BQO8ZF18ZDX9vbkudXVqDe1tBRtjZF8ren8jw0mOweLRFvNHz8+/fzCQvQ+Llwwbx+h2lwY9YqVACUinxGRH4rI8zbOR/6ps+A27oGMRlFQmpmZzPFsbExvj2E6/+OD+H288sr087u7wBNPAK++Gs57aYxrd8gBWz2o/wzgYUvnIs80yVCrksGXN/8zGoW18WD8PhYWDn7v5MnpoFxbKItU62T8EVVkJUCp6h8D+Esb5yK/NM1Qqzp0t7Y2HbjiIFWWnu1TCaXkHFS8y2/sp37K8ORM36Y6QvmFpqLW5qBE5LSI7IjIzpUrV9q6LBlK9mo2N6eH6+LAkg4O43G9obu68z++rTmKe1Dp4ARMz8N5zZcbG4cKzfTsF5rWApSqPqmqi6q6eOzYsbYuSxYUDddlBYuzZ6PehIuhO1/WHKWvc+5c9DVOqx+Po4C1vR3IfJovN7ZQhwp9CfA9wzRzKpU3XLe+Xlxvb329eep2Hh/WTq2tRe872YM8exZ47rnpdsRzTyHNp1FDvgT4vlFVKw8A9wJ4vsprT5w4oRSG8Vh1NIp+jR2NouPl5cnxzZuT4/gRv851u5LXdH295HXTn0fy+ObNg683kt2fiB42tXWdLszOZr+v2Vl71/Dl8/OlHSUA7GiFWGErzfxpAM8CeI+I7InIv7BxXupeOtMuXtOzvJzfM3Ddk+li7VR87rgHt7ycPSc3k/ofZfw5+DYnE+JQFns3wbKVxfeoqt6jqodUdV5Vn7JxXvJDnGkXz/9sbUXPnzs3KWeUtLLiLlh0sXYqPc+WxVlQ9m1Ohjd7v/n2C40hzkFRJRcuRMFpfT063tycDkxnzkTPxQFrext49ln7N20btfPqSCZlAJOagumgvLrq+f5RZYa0v1Sf+Z5MUhMDFJVK36TX16dr5wFh1c6rIy8pA4iG+TY2pve8CjZI9ezG1joGeCe4YSFVkhxaS1tYmC6UGt+4g7xR51Cdnl9Kvsf4szl6NPD9n6oIcfPAENvcc9ywkKyKC6Im3bwZDbWlq3ing1NbVR9cXScrKSOpavULZ0JMXGhTz+ZlhqT3Acqnkjghi9f6JJ09C/zGbxysQZdMKGir6oOr6+QlZWxtTV+v095iUeKCSdDKCnx5fL7Z+5ZoQpX1OkD5VhInVOmbdNxz2twEFhcnVSPSGXXjcTtVH5JzZLav05eCtrfUybYreq3Nmz17gJSnymIp2w/bC3XTiyHH4/IFlW0t7OxK1mdi4vz56c8t/jwffDD7+fPnp49dL+Stc50mn43tz7NQ3YWlRYszTRZscpEwOYKKC3WDT5LIKjsTT1ifP39wYr/NkjhdKfpMTHqPycy55HHe88njZILBeOzm869yHVefjVVlk/qm26JX/T/fVnIBkxgGZxBJEmVDO0D5nkR943q4K+u4qBp5VoKBiwW1Va7j8rMxkh7iKsNFsTQUVbpZth82h/iKhnbaGl7yjS/v2+Ywa9EQW53r+PLZTKk7PFd3SI9DfOQZDGWID8ge2gGmJ/bTlbaH0JNqY1itjI0htSrnqHMdXz6bW+pcPD1+mvV9IH8YcHa2ekKDjXNUwSG+wak6xNfrHlTexH48gd9XvvUSTBIM6vaOyq6T99mkq5C3ymYPKkRtVBsnr6BiDyroAFXl5tVq9lXH0sOay8v9yF60FXCT51lYiIJS8vjcOTftz5R3Uy56xDfsvgUoGpyqASroWnxNCoeGPKynmp8tlxziSm4/fuHC9GdSdk4fxX+vyWzMJkO0IsCRI5PSTGfPRtUxvva16PjBB1v8PKomOmjGEBfrvtFQVIlith9trIPqm6LhyrJNBeOeVahDnqY9qPTr3nrL7RBo3rq8KXV6TEQ9gzY3LOxaUZpzH6hWS6dPbqK3tXUwGcTLFOt96evHx8l2Ntn/KauayK/8StSTSrKVNJN1vVOnokfyuUJxiOqyFA+rO5APqkQx2w9u+V5fVi8inkeJv5/eej3rN3mfkidiZT27pj2/ojnKhYXpzyGer0v+bF1FPdn0nGCjuac2cZ4rW92EDiaAZMIQkiSGZjw++O8860ZYFHzS5+g6OFXN0ms6jJsX2JPnX1qaDiAmQ59Z1zt58mBArBWgurihMUBlq/u58HPMxADVM0U32qyeQN5N3scelOt2pYPyuXPTc3NxcF9aspP1mL5e1i8P19Egi6+uKr+9N8kmtHXtEDFAWcEA1SNFvYys4JT8mayirT4Wz3XVs6uy7imrB2oSnNLXO3Pm4C8TyeHZW2wHqCrnqhucqrSjLOhV5WOQY4CyggGqZ9LzMDdv1p9HMZnLKTo25aoHVXeRr2mALJuDSj/+6lDN3ktdXQUoW+/Dx5s7A5QVDFA9lDVUV7c3VDfYuE5Nd92zq9J+mwEy63onT6oePz59/oUFtR8Y0lwEqCq9Fwao5q8fCAaonmtjTVNbw4JtBMG8YxfvMX3+9Jq0+PzeB6im+hygmMVnRdUA1YtisaFSLd5HyfXPV71GG3tqtfFe8rjeIyrv/BubNd9g3SKtVYqwuijUWvYXV/W8LCLbW1WLxTJAdSSIjfP2qXpW/dsB1wEy8/wzDS5Q5/9rlWrkLiqWM0BRiUFsWBgqVb+rOiTFbUtyseFg1+J7YTwGkz62cf7keVoJ8DduZA+QJQNPldfUVVQTsE69wLzXuqw5yAoaXmGA6kBc/DQu2TMz4+c+VcnA2aTMUGjW1qKSRCsrk/v0ykr0nGmvNqsE0huHelrcNS/o1Q18LoJnmbwivtzFuBNBVzMPma0K3S41qRYfKlXg1VeB7e3oEdvair4uLVWbtsk7d9xjBhKbZ755I/uXEt8+2LxhwJjtDQyJ9nEOqiNtJR/Y0GUCQ1vW1qIABUyCUuzMmcnfU9N5wlp/3y7nXprMOVX5y+5Ld5rzXq3gHJTH2h46S5+v7vnrVouvez3T9pmKezjpwJRmMk+Y7H3Gcn8ZcTn30uYQFudzyBCH+DrQ5tBZ29mCda9n2r6mvbvk60SijQtVs4PUE09EDyDa7PDIkWbDfFnJJplBqi/DZZzPIUPsQXVkbW365hQHKZtBo+1swbrXM21fVuLB6mr5Z5j3c5cuTb/u/e8/+LO7u8D16/U+u6ElmwSti8xByldlNW/ZA8DDAF4C8DKAXy97PStJtKftCuZ1r9e0fU0rQBT93DveMd2OM2dU3/726ecyi7xW4M1uxvn5dQdfW6fSuem1aFDQVqkjALcB+N8A/jaAHwHwHICfLPoZBqh2tb0HVN3rNW2fjeCWDDxAVJbo5s3iIq82SiBlHbeiTtCoGpzyyvYwQDnhxb8jQ1UDlI0hvpMAXlbV76rq3wD4XQCPWDgvWaA5cx/qaFip7vVM2peZePCZuahCQ8HEfNbPffjD0bDbxka0Lm1jI0otf8c7pl/XdA4qvm7RcStsDWElw01f5swC0HRYO1Q2AtS7AHwvcby3/9wUETktIjsisnPlyhULl6Uy8T/eNrMF61zPtH1ZwU0qTMxn/dz161GiRDJoLC0BP/jBdNuazEHZlvU5Vtbm4lfO51ilGk4FGltsZPFl/R544KNS1ScBPAlE66AsXJdKtL3Qtu71TNqXDm7x4lds5v9M0c8lF9HGna477jD87BzUuTPJelRteT0be1ZWJf8Nbm5O/s36un7SBuOFuiJyCsCaqv7M/vHHAEBV/13ez3ChbrvavjHVvV7T9mXdrAsLsO7/W69zkzf67Cwv+iwKrmU3qUaBzUL745claxuWnZqKqYZfvLnqQt3aSRHpB6Je2HcB3IdJksTfLfoZJkmQsToZZhkT861MNDtIEmiSGJLOXLx58+BxJsO9jM6fV11amuz0HO+NtbRULXuxD8kAtrWdlesK2tywEMCHAHwHUTbf42WvZ4AiY3WDUxeZYwVtMbn5Nsl6zLuxxcEqK2CYpMant7tfXj54XNRuF2n5oQe8tjYQbUOrAarugwGKjNUNTl3sYFrQHpMbf9PfoG/enP65ZDp9fI7kw/RmmA5SVYOTixuxN+vQDPXlfTBADdGQtpf2qaeUp6CNTW6+JjfurMAWP97//ihYpYfg6gTDvN5Jurdn2uNruv6sLz0P1fB7gqoMUMMUwk3bFhc9KNsBPud849nZxjffJr9BZ81BZfVqsobgqgwn5rXp3LlmPahku+sGtiqfgWnAI3MMUEPEAGX2GbT4+ZncfJv8Bh0HkWSCRN4jGZzKbupFvZO4OkdRACx6j7YDStsVVSgfA9QQDSlAGWbx1f55y+35q0OzuTdfV0M4Wdl7WT2cunNQecHk3LlmWXwuhuTYg/JL1QDF7TbIjIPFqJWYbK4H2N/yoeb5Dr/5WuZC4SNHokoVLrZHmZmZXhidZ2UlKvVUdRF10e7QcZvj129sTB9nsb3APP4MyxZnk4eqRDHbD/agHOmiB9Xkmi6TOaq2x6T3Vfe6OY+sOZs2JvOTvZmyIbgqvTlXvRObPcm+ZL/1BdiDGqDZ2fzejE9sbWSX13sLRHovsLjHBLgtZROf5447onqDS0uTng0AbG9H38u6Xvo5l70Tm8V119aitmZ95uQv41JHTbDUUY80KYdjqwRQnbtLesjR5GcBO8Ex572qZpeySd5gbYmbYFKKqO1dmyl8VUsdsQdFfpmbsz93ZfpLWFZ7HPXc4pt70upqVGn97Fn7N30bvRT2TsgVbvlO7iT2YKrM1ZDd3Nz0/lBVuRoezThvcrhsYSF6bmEhOj5xot1tFdLXKDtOq/oR1z0vDQsDFGVL39AzNv4DUHwD92l+qKgtRakMTXpzVVIkMs6bzF67fHmy/xQQfW1rW4WyTfFsbZo3tM33qD4GKN9UDQyur1U1kaHJDdy3pA2PrK1FQWhmJmO34BaCk2rxpnjjsZ1N88quw54UAWCauXfaTBVvkBp9qx1VF6a6fJ912mD7c3X899TlwtKya9tqGxfPDhcqppkzi883lje5a3ytIlojnSyvzTbfZ1lbytrb5HN1uEA52ZuouzGhLZqTSVj1+7auQ/1UNYuPQ3zUjbxhvlCG/27csDdnlZJXSWE0alZJIS0dj7OOszIJ49eVfb9OO2ych3qsSjfL9oNDfAVCGeKr8rq6lSGaVpgwGZL0eCsSFzX5yioqlNXBS9fza1rtom9bYFA9YCUJMmJSlaLpr8C2KkxkcVkX0BGblRSA6K8lTkwADg4dxiOhRXXw0vX8mtbJs11vj/qJc1C+abP4qsm1XMyVNT1n0c+5LlobmOT8VixrXis9bVflGCh+TV576v4MhY9zUKFyOLdh9Vo+zSEVtaXt4NTmMoEcRXNMyZ5KLCvpou7xhQvN1jTZ7CUWvW8KEwMUNdNmIAWKb/ZttyVPnfVjjpQtfo2Pk0wTE5JDh12taeKi335igKLw+FShIqnjdpUFivF4es5pPI6+Jl/fRNwrW16OzjUzE31dXvZjcTF7UuFikgTZYWPuLC8xoy/Sd2rLQ5DJ4bu87TpcJSZcuJD/vOteTJX3TWFikgTZ4VPSRB1lgbVO4G26UtUyLVn8ajsxQTXahXdr6+D3lpejvabaCBJl75v8wSQJoirK5ow6nlOqq8ock+30dR+4mFuj7jFAUZg6zJLzVXLuxeYcUxmRaAfe5eXp55eX83fmtamr903ucQ6K/FVnTsqHHk0Hc2jJ4TkR4MiRbha/nj8fDfNlPe8aF/32FwMU+cvGHE+b8hIeHLU7a6v169ejINXm7rZxD2Zr62Bx27Z21+Wuvv3EIT6yw6eFu76x/NnEy7zi1OqVlelhruvXi+ecbHNd3LZOO4qOKTzM4qNwVKmbU5fNLL4WJHtNwMHsuS5Tq1m2iKqqmsXHIT4atrIg41Edv6xir2ldDmuxB0O2cYiP+mEAQ4nJobO4YkN67RGz1rrDWoD2MUBROHwqCtuRrGKvy8vAzZtMre4SawG6YRSgROQXReTbIjIWkdLxRCIjvhSF7ZAqcOrUwedWVycp5kytbhdrAbpjOgf1PIBfAPCbFtriN88my2l44pJC29vR8fJy9NwTT0yOL16cLvdD7nVVC3AISSlWsvhE5KsAflVVK6XmBZnF10ZdOKISa2vAq69Gf07OPy0tAc8+G/4NKuSbbpu1ALPWwK2uRr3nEIYVvavFJyKnRWRHRHauXLnS1mWJemVtLSq+urEx/XwcnOJRz1j62Gchz+O0WQtwSEOKpQFKRL4iIs9nPB6pcyFVfVJVF1V18dixY81bTESZN8Pz56P5qXjhbjwkeOqU/zf5kG+6bdcCzMrmjK/dt+oZpXNQqvpQGw0h8pZH84/pm2GyrNDCArC7O5mjAibDgEtLfg+XhbynUxe1AONrxJ8T4P/n1ATnoKriHNRwefZ3nzf/cORI9Hx6bVSbezKZCnlPpzbnz5K/qMRCCOaxqnNQUNXGDwA/D2APwF8D+AGAP6rycydOnNDgzM5mJThHz5P/TP7+spPbo4fL6xYYj7OPx+ODl0q/1vSaeV9tnH80mm77aGTv/H2R/Jzizyd97DsAO1ohVhilmavqMwCeMTlHMJhKHrauNh50dN2sskLxnFPayop5DyrutR05EhWjXV8Hzp6dHJtmjxUNXQLh9AzaMKTtRViLj8Lk0byQD+LgFA/vxZsHbm1NnmsapJIJDPE819e+Fn2Nj0cjsyGt5E13fX36ppvcPoQiQ9lehEv6KEyBbcXuWryr7dLSZM5pYyP689KS2c62yayx3d3oueRXW3Mfa2tRMDp7dnLzXV+Pemi+ZyF2YQjFebndBoWpbuKCSaKDSW+t5QSL+JTxZdPHpufOqlJhK5GhaJgvpAQAKsftNohsCWjI0NVv1VkLUWOrq3aCR8ip5uQGh/hoGLra8bcHOw0nezYLC9Fzya82F6RmVWsPKTilPwOuQDHDHhQd1McEhK7aHernlZBMYDhyBHjwweksvgcftJc9llcyKIQgFXp9PB8xQNFBISQgzM7mB1ETfQzOFiSzxuKvyRux6zkowO8glcx0BA7On/lcxcNnDFAUFtcBJITg3JFkSnPWVxvnD3V9D+fP3GAWHx1kK/PMRTBxnRXnWVmjIWqzZJBtIZdqapN3223QANnqjczNRf/L+T99EEJd39PWlhtDSsRggKJuxAFHJApAacmgxOE18lxbW26EvGdWE5yDooNcJSDkyboWgxIFpI35syEmYnAOityp878l/e+wadE4Uz3M4gt5Tic0rj/r0LfZiHEOiobFVu/uxo3sjTUCDU5NhoSGNMdhm+v5s9AXMtfFAEXutFEtIfAA4lJySKjqNupDm+MITVuJGL5ggCJ3kr0RFwIqF9SFZBXyzc0o/bmo8Koq8Oqr0wFtZaU4oFF72krE8AkDFHUvK9AU1bBz2WtKZg+WZRoGoM6Q0IULwKVLk/p6MzPRXlLHj3NPJh/kJWKMRv4vZG6KWXzUvaxA09WQXUuVJNpKXKha2y7uPX3jGwfP8corkx5UH2+CIRnKRoUx9qCIWtbWPE+dISGRaIPDM2cOnmdhod83wdCEupC5CQYoakcPtp2wMdzXJHHBpLl1h4SyntvdnexyS9QmDvFRO/qUZWcw3Nd2UdG6Q0KXLh18bmGBc1DUDfagyE89S1ZIanstS5UhoThjL56DWl6OHkDUg2IWH3WBPSjyU1fbXuSVebLIx035RIA77gCWlqLHxsbke9vb0ffYg6K2sdQR+cmHbS8ctKFoUz4fStbEbytuQ/rrVePDAAAGu0lEQVSYyIaqpY7YgyJqke+b8g0pQ4z8xx4U+cmHHpTDwrEs4EpDxh4UkSmHmYfsqRCVYxYf+akP66aIyAh7UOSnPq2bIqJG2IMiIiIvMUAREZGXGKDIDz2uHEFEzRgFKBH5pIi8KCLfEpFnROSorYbRwHRVOYKIvGXag/oygPtV9X0AvgPgY+ZNIiIiMgxQqvolVX1r//ASgHnzJhER2Zde383it/6zOQf1GIA/zPumiJwWkR0R2bly5YrFyxIRFWtrk0iyqzRAichXROT5jMcjidc8DuAtAJ/NO4+qPqmqi6q6eOzYMTutJyIq0eYmkWRX6UJdVX2o6Psi8ssAfhbAT2sXhf2oH/K2uWDlCDLU9iaRZI9RsVgReRjAOoAHVbXyuB2LxRJR21SBmcSY0XjM4NSVqsViTeeg/iOAWQBfFpFdEflPhucjIrIub5NIjvn4zagWn6r+uK2GEBG5ULRJJMBhPp+xWCyFz+G+TRQ+3zeJpHzcsJDC58PmhuQ9bhLpj7bmoIiIgsBNIsPDAEVERF5igCIiIi8xQBERkZcYoCh8edUmWIWCKGhMM6fwMZWcqJfYgyIiIi8xQBERkZcYoIiIyEsMUERE5CUGKCIi8hIDFBEReYkBioiIvMQARUREXmKAIiIiLzFAERGRlxigiIjISwxQRETkJQYoIiLyEgMUUc+oFh8ThYIBiqhH1taA1dVJUFKNjtfWumwVUTMMUEQ9oQpcuwZsbk6C1OpqdHztGntSFB5uWEjUEyLAxYvRnzc3owcAjEbR8yLdtY2oCdEOfq1aXFzUnZ2d1q9LNASqwExibGQ8ZnAiv4jIZVVdLHsdh/iIeiQe1ktKzkkRhYQBiqgnknNOo1HUcxqNpuekiELCOSiinhABjh6dnnOK56SOHuUwH4WHc1BEPaM6HYzSx0Rd4xwU0UClgxGDE4WKAYqIiLxkFKBE5N+KyLdEZFdEviQi77TVMCIiGjbTHtQnVfV9qroA4PcBnLPQJiIiIrMApao3Eoe3A2AiKxERWWGcZi4inwDwzwFcB/APC153GsDp/cO/FpHnTa8dsLcD+IuuG9Ghob9/gJ8B3/+w3/97qryoNM1cRL4C4HjGtx5X1f+aeN3HABxW1fOlFxXZqZJi2Fd8/8N+/wA/A75/vv8q77+0B6WqD1W85u8A+G8ASgMUERFRGdMsvp9IHH4YwItmzSEiIoqYzkH9exF5D4AxgP8D4F9W/LknDa8bOr5/GvpnwPc/bJXefyeljoiIiMqwkgQREXmJAYqIiLzUWYAaepkkEfmkiLy4/xk8IyJHu25Tm0TkF0Xk2yIyFpHBpNuKyMMi8pKIvCwiv951e9omIp8RkR8OcR2kiLxbRP6HiLyw/29/1HWb2iYih0XkGyLy3P5ncKHw9V3NQYnIXFyJQkSWAfykqlZNsgieiPwTAP9dVd8Skf8AAKr6ax03qzUi8ncQJdf8JoBfVdXe778iIrcB+A6AfwxgD8CfAnhUVf+s04a1SEQeAPA6gN9W1fu7bk+bROQeAPeo6jdFZBbAZQA/N7C/fwFwu6q+LiKHAHwdwEhVL2W9vrMe1NDLJKnql1T1rf3DSwDmu2xP21T1BVV9qet2tOwkgJdV9buq+jcAfhfAIx23qVWq+scA/rLrdnRBVf9cVb+5/+fXALwA4F3dtqpdGnl9//DQ/iP33t/pHJSIfEJEvgfgn2LYhWYfA/CHXTeCnHsXgO8ljvcwsBsURUTkXgAfALDdbUvaJyK3icgugB8C+LKq5n4GTgOUiHxFRJ7PeDwCAKr6uKq+G8BnAfwrl23pQtn733/N4wDeQvQZ9EqV9z8wWVsHDmrkgAAR+TEAnwOwkhpJGgRVvbm/A8Y8gJMikjvUa1wstqQhgy6TVPb+ReSXAfwsgJ/WHi5Iq/H3PxR7AN6dOJ4H8P2O2kId2J93+RyAz6rqf+m6PV1S1Wsi8lUADwPITJrpMotv0GWSRORhAL8G4MOq+v+6bg+14k8B/ISI3CciPwLglwB8oeM2UUv2EwSeAvCCqq533Z4uiMixOGNZRH4UwEMouPd3mcX3OUQl12+VSVLV/9tJYzogIi8DeBuAq/tPXRpYFuPPA3gCwDEA1wDsqurPdNsq90TkQwA2ANwG4DOq+omOm9QqEXkawD9AtN3EDwCcV9WnOm1US0Tk7wP4nwD+F6L7HgD8a1X9g+5a1S4ReR+A30L0738GwO+p6r/JfX0PR5aIiKgHWEmCiIi8xABFREReYoAiIiIvMUAREZGXGKCIiMhLDFBEROQlBigiIvLS/wfTsHpUDS+UswAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "np.random.seed(0)\n",
    "X_xor = np.random.randn(200, 2)\n",
    "y_pred = np.logical_xor(X_xor[:, 0] > 0, X_xor[:, 1] > 0)\n",
    "y_xor = np.where(y_pred, 1, -1)\n",
    "\n",
    "plt.scatter(X_xor[y_pred == 1, 0], X_xor[y_pred == 1, 1], c='b', marker='x', label='1')\n",
    "plt.scatter(X_xor[y_pred == 0, 0], X_xor[y_pred == 0, 1], c='r', marker='s', label='-1')\n",
    "\n",
    "plt.xlim([-3, 3])\n",
    "plt.ylim([-3, 3])\n",
    "plt.legend(loc='best')\n",
    "plt.tight_layout()\n",
    "# plt.savefig('./figures/xor.png', dpi=300)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_data = X_xor\n",
    "y_data = (y_xor>0).reshape(-1,1).tolist()\n",
    "feed_dict={X: x_data, Y: y_data}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trains the same 2 -> 2 -> 1 NN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.7305015\n",
      "4000 0.46023026\n",
      "8000 0.45174533\n",
      "12000 0.4499272\n",
      "16000 0.4488317\n",
      "20000 0.44807425\n",
      "0.795\n"
     ]
    }
   ],
   "source": [
    "# Launch graph\n",
    "with tf.Session() as sess:\n",
    "    # Initialize TensorFlow variables\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    for step in range(20001):\n",
    "        sess.run(train, feed_dict={X: x_data, Y: y_data})\n",
    "        if step % 4000 == 0:\n",
    "            print(step, sess.run(cost, feed_dict={X: x_data, Y: y_data}))\n",
    "            \n",
    "    print( sess.run(accuracy, feed_dict={X: x_data, Y: y_data}) )\n",
    "    y_pred = sess.run(predicted, feed_dict={X: x_data, Y: y_data})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Can't fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X+MXNd1H/DvocqYgTLkSioFyl4bEhDDdiEYaywthmggto3TCEFgJQECRCjaGApAFHC0QzIBEscAd9jaaAs7S+6qBhoZMpoAjgIDrlwjUfwLrZ0YMWkvjY0rm5IhBCi8bRQzjChaaJVYOyd/vH3aO2/fzLwf9+d73w8wkGa4+96d2Zl35t577rmiqiAiIorNgdANICIiKsMARUREUWKAIiKiKDFAERFRlBigiIgoSgxQREQUpdYBSkQOicjXReQvROTbInLBRsOIiKjfpO06KBERALer6isichDAVwEMVfWyjQYSEVE//aO2B9Aswr2ye/fg7o2rf4mIqJXWAQoAROQ2AFcB/DiAj6nqlZKfOQ3gNADcfvvty29/+9ttnJqIiBJz9erVv1HVo/N+rvUQ38TBRBYAPA3gMVV9dtrPHT9+XDc3N62dl4iI0iEiV1X1+Lyfs5rFp6o3AXwZwEM2j0tERP1jI4vv6G7PCSLyowDeA+C5tsclIqJ+szEHdQ+A39udhzoA4FOq+kcWjktERD1mI4vvWwDeZaEtREQ0xQ9/+ENsb2/j1VdfDd2Uyg4dOoTFxUUcPHiw0e9byeIjIiK3tre3MRgMcO+99yJbfho3VcWNGzewvb2N++67r9ExWOqIiCgBr776Ku66664kghMAiAjuuuuuVj0+BigiokSkEpxybdvLAEVERFFigCIiokoeffRR3H333bj//vu9nI8Bioiog4pFgmwUDXrf+96Hz33uc+0PVBEDFBFRx4xGwNmze0FJNbs/GrU77oMPPog777yzbfMqY4AiIuoQVeDmTWB9fS9InT2b3b95005PyheugyIi6hAR4OLF7P/X17MbAAyH2eMpJQKyB0VE1DFmkMqlFpwABigios7Jh/VM5pxUKhigiIg6xJxzGg6B8Tj7rzkn1dQjjzyCkydP4vnnn8fi4iKefPJJew0vwTkoIqIOEQEWFibnnPLhvoWFdsN8Tz31lJ1GVsQARUTUMaNR1lPKg1EepDgHRUREwRWDUWrBCWCAIiKiSDFAERFRlBigiIgoSgxQREQUJQYoIiKq7bnnnsPJkyfxhje8AR/96EednINp5kREVNudd96JjY0NfOYzn3F2DvagiFpyse9O1/A18uzw4SyvvHg7fNjaKe6++268+93vxsGDB60ds4gBiqgFV/vudAlfowB+8IN6j0eKAYqooS7tu+MKXyNqg3NQRA11ad8dV/gadcvHPvYxfPzjHwcAPPPMM3jjG9/o9HzsQRG10JV9d1zia9Qd73//+7G1tYWtrS3nwQlggCJqpSv77rjE16ibXnzxRSwuLmJtbQ0f+tCHsLi4iFu3blk9BwMUUUMu993pCr5GgQwG9R5v4NixY9je3satW7dw8+ZNbG9v47DFLEGAc1BEjbncd6cr+BoFYrknE4pogK8wx48f183NTe/nDcXcl6XsPqWNf9/5+Bq1d+3aNbzjHe8I3YzaytotIldV9fi83+UQn2NcA9J9Xdh3xzW+RnaE6FC00ba9DFAOcQ0IEdly6NAh3LhxI5kgpaq4ceMGDh061PgYreegROTNAH4fwDEAYwBPqOp62+N2AdeAEJEti4uL2N7exvXr10M3pbJDhw5hcXGx8e+3noMSkXsA3KOq3xSRAYCrAH5eVb8z7Xf6OAd1wOirjsftgxPH9IkoVd7moFT1r1T1m7v//wMA1wC8qe1xu8LFGpCY5rVSKwKaWnuJ+szqHJSI3AvgXQCulPzbaRHZFJHNlLqobbhYAxLTvJaLQOkygMQU2IloPmsBSkR+DMCnAZxR1X1J+Kr6hKoeV9XjR48etXXaqE1bAzIcNl8DYh5jfT0bOswDoM95LReB0mUAiSmwE1FFqtr6BuAggM8DOFfl55eXl7VPxuPZ95seM7usZjcbx2zShuFwsh3DYbO2mMfKj1G876K9KyuTxw7xOpIHg8HkHz6/DQahW9ZLADa1Qqxo3YMSEQHwJIBrqrrW9nhdZHsNSCy1zWwWAfXRMyxrr4lDfh3Wkf2R+sbGEN8/BfCvAfwLEdnavf2sheNSiZhqm9kOlK6rXpe1d2MDOHOGQ35EMbKRxfdVVRVVfaeqLu3enrHRONrPxbxWE64SQFz1DMvau7KS/dvGhr+5PGYRElXHYrEJGo0m1z3lQcrnOijbRUCLAeTixb37QPvnV9beS5eyf9vY2Ps5l6/jaJT1zvJz5M95YYHDikRlGKASFUNtM5uB0kfV62J7y5w96yZImVmEwGQAHg650JqoVJVMCtu3vmXxUXUuMh6nncdH1qBpZ8de1iPVxCy+qKBiFh97UBQVXz1D1z22suG8c+eAI0cmf441GT3pyP5IfcMARTMVh55iG4pq0z5Xc3mzhvOWliZ/1tWQIlEXcLsNmir20kA22ueixzZtTdfSErC1FX55AFEqGKColNkLiLE0UOztK1vT9d73hl8eQJQSDvFRqdj3soq9fWVrul5+GVhbC7s8gCglrfeDaqJv+0GlTB3sZWVTjO2btaYrlgBKFJK3/aCou2Kp+Wcyzz0eZ2WKTKHbB8RT7YModRzio1KuKzs0YaZuj0bAZz+bJR2cOAH8+Z8Dy8th21dsa+hqH0SpY4CiUj4qO9RhJkXkPaStrey/DzyQBc+trSxT7siROAJBDNU+iFLGOSiaKaZ1UGavrsxwmCUhHODANVHUOAdFVsTUC5i3n9PFi90ITqx47tjhw9mbqXg7fDitc/RABz7O1BdlSRumGBIk2op9cXQn+Ni8kBskWsE5KEqCObxn7uME7N2PJUGiKVY8J5rEAEVJKCZtXLiwF5juuANYXd37mVQv4rEvPibyjUkSlBSzF5G/dc37XbiIx7j4uFNmvZi2roc+zpEwJklQJ5mf+3zeuezfUlVncTSTKTqKCRavY4AiikRxcfSsiudMpmhhMKj3uO9zMMHidZyDIopE1cXR3pIpDh8uvygOBmlvAOij7Sm/PhHhHBRRZKosji5btGw9mYLzKGH04HWvOgfFAEWEuCpmVOU8maIHF8oo9eB1Z5IERSGFifwU53NirDRPZBsDFDmTwoU/9p15y9RJpuiEvmW1+UjiSASTJMiJVKoipLg4NrZK8871LauNCRav4xwUOVN1Ij+G+R8b8zm+n4fz89XJ4nOZ8deDOZm+4RwUBVdWfbwYnGIYBrQxnxPieTivNH/rVvZEireygNO3Xg55wQBFzsy78Mcw/2NjPieG50HURZyDIidUgTNnsorj+bDemTP7K46Hnv+xMZ8jku3iu7Q0+Txi2t03GdOGCqmXGKDIiQsXgCtXsorj5jDfiROTF/48IJjzVL6TE0ajyfmbvE1V26AKvPzy3hb0ua0t4MEH40kISUKd4NTDrLa+4RAfWZcPeV25svfY2bNZb+rEiWxrDPNnY1jP02Y+RyTbav7YscnHl5ay53HhQvv2EarNhVGnWAlQIvIJEfm+iDxr43gUnzoLbvMeyHCYBaUDB/bmeC5dmqwp14X1PPnzePHFyce3toDHHwdeeimd59IY1+6QA7Z6UP8VwEOWjkWRaZKhViWDb9r8z3CY1nqe/HksLe3/twcemAzKtaWySLVOxh85lUL1lqqsBChV/VMAf2vjWBSXphlqVYfuRqPJwJUHqXnp2TF9CM05qHyX39xP/ETLgzN9m2qIYdmGTd7moETktIhsisjm9evXfZ2WWjJ7Nevrk8N1eWApBofxuN7QXd35n9g+hHkPqhicgMl5uKjF0lPjUGFjnVzuoKpWbgDuBfBslZ9dXl5WSst4PDl2Mx5nj6+uqg6He/fH4+z+qVPlj6+utm/HcJi1IT9+8b4PxfPs7KiurEy2o3i/kfKBs+xmk6/zdNVgUP7aDQZem2F+HvKbz89FVQA2tUpcqfJDlQ7EANVZ0970Ozuzg8XOzv7juGyPrw+h16DMAJWGiF6/aV8mY8IARVaU9VDMnoHZc/AZLEJ9COf14KwHZQao9nz0biJ5/UJ/eauqaoCylWb+FICvAXibiGyLyK/aOC6FV8y0y9f0rKxMz7RzvdBWKyZg2D4nsDcnt7JSPid3oPCJav06xDYnE8tcVR09STTJPxepL9sw2crie0RV71HVg6q6qKpP2jguxSHPtNPdSdiNjezx8+f3yhmZzpxx92EI8SEsJmWUcRaUY0vf7snFPkVdWbZhYqkjquTChSw4ra1l99fXJwPTY49lj+UB68oV4Gtfs/+h8L0XUh6UzRqCZUH57Nl494+qZDCYvl0GJaNt2a7YMEDRXMWL9NraZO08IK3aeXVMK2gLZMN8ly5N7nmV7MWAC2rbiSjAO9+GxSNuWEiVmENrRUtLk4VS8wt3yh+MItXJ+SXzOeavzcJCugsiK0tx88AU29xx3LCQrMoLopp2drKhtmIV72JwKl4DXM5PuThPWVKGqWr1C2dSTFzwKbZEE6qs8wHK18Wx61SBc+cmHzt3Dvid39lfg85MKPBV9cHVeaYlZWxsTJ4vaG9xVuJCm6BVFvimifliH1uiCVXW6QAVW0mcVBUv0nnPaX0dOH4860GVZdSNx35Kr5hzZLbP07nMqDrZdrN+1ubFnj1AmqbKYinbN9sLdYuL0MbjeErihFL2mrTRtHqCr4WDdc7T5LWx/XrOVHdh6axFom0WjnKRMDmCigt1k0+SGI2yb8lm4dJ8wnp1df/Evs/txEOZ9Zq06T2amXPm/WmPm/fNBIPx2M3rX+U8rl4bq+ZN6rfdFr3qZ95XcgGTGHqnF0kS84Z2gPl7EnWN6+Gusvuz0lrLEgxcLKitch6Xr00rxSGuebgolvqiSjfL9s3mEN+soZ1U6lLZFsvztjnMOmuIrc55YnltJtQdnqs7pMchPooM+jLEB5QP7QCTE/sXL+6/3/WelI9htXlsDKlVOUad88Ty2ryuzsmL46dl/w5MHwYcDKonNNg4RhUc4uudqkN8ne5BTZvYb7snUexi6yW0STCo2zuad55ZW4cEY7MHlaJI9lIif+B7u406N1sBqsrFy2v2VWDFYc2VlW5kL9oKuOZxlpYm97NaWlI9f95N+0tNuyjPuuUX7K4FKOqdqgEq6Vp8TQqHpjyspzo9W84c4jK3H79wYfI1mXfMGOV/VzMbs8kQrQhw5MheaaZz57LqGF/5Snb/1CmPr0fVRActGeKKqO4bkVNVopjtm491UF0za7hy3qaCec8q1SHPtj2o4s+99loEQ6B1ekxEHQOfGxaG1qXqvWVUq6XTm5vobWzsTwaJMsV6V/H8+X2znU32fyqrJvLrv571pEzWkmamVUWoWjIob6Rq2FI8rO4w1bT3qq2fJ0OVKGb7xi3f6yvrReTzKPm/F7deL+tZBu85lJjXs2va85s1R7m05Oh1qDuvFGtPivNcpeq+F1MdtXANfUiS6JvxeP+1ojikN++iWzxG6OBUNUuv6TDutMBuHv//H7SYReYiQIUIUgxQ+9Rd19f3cmuzMEB1zKwLrXmblbkXaw/KdbuKQfn8+cLxbV6MXQWoJm2pkr7dJJvQ1rkTVPe9GutnLjQGqA6Z9U2sLDiZv1NWtDXGb3OuenaV1j2FDFB1ftdmW5q2t0o75gW9qiINcnXfq7GNWsSgaoDqRJJE15Wl06+t7d+Hqfg75iZ6bbaNUJ19vy1VN/X68uOWJVicO2f/eTRSpwZfKmzVCpy1z1Ugdd+rrt7bvVElitm+sQfVTNlQXd3eUN25HNeTvK57dpXab7PX0mTIzFbPpchFD6pK78XW87D5WljAOSh70IeFun1jVg+vu0C5eIxp902qe6npwP56hqrtv/i3eS5VjEaT7cyP76zDUpYanmrviF/zJ9R9r7p+b/dBJ4rFpqp4ga97wW/7+1XP4WNPLR/PZSrXRVHb7t/UtD1Vxm1dFGqtss6r7XECBs+679Wg7+1I9WI/qJTZ2I7exwJl81tfzkUPJOhi61u3ygeSbC2UnXb8uuoGuWmlj8zHq/xMzxT/NMX7dd+rXS8k4BIDVADm0FmMVR1MedtMnORNRJXA6yI4zwpudQJfgOBp44sj2cMAFYCZQZeXJopxnyozcDYpM5ScWWWK2pb4KTt2V00LenUDn+uebUFKXxz7gnNQAalGtnFeCRsbDibD1txJ3WPPG0Oy1Yam5s2h2d7AMCBfc659V3UOigEqkJQ+CL2Y5K2TyNDkghxLgGqSEFLlj92h7kUKXxxTxySJiPkeOps36TtP3Uneuudr2z4r6iQguF4o6nLuxefi1wQronPONS4MUAG0qepQl+9J37rn46R0Cc9zL85EWAlilt7NuSaAC3UD8bGA1MdC2zbna92+puuXbK1Lok7hwtoIVSk3Me8G4CEAzwN4AcBvzft5ljryx3c1Za/VnqfnijX/vaq3umIpfFrnOdUp29T2XBHpww7doaFiqaPWSRIichuA7wL4aQDbAL4B4BFV/c6032GShF/qedK37vkat69pMoGNJ5/qeI+tZA3TtB5rbNmIHVE2EpFa785nksQDAF5Q1b9U1b8H8IcAHrZwXLIgH1c3uRxPr3s+Z+1LaGLeK1sJGCnPjSWsb3O2NgLUmwB8z7i/vfvYBBE5LSKbIrJ5/fp1C6eleXxP+tY9n9f2VZ1zMi+8XeQzAYNllKxS7d9CYhtJEmWdy30vlao+AeAJIBvis3BemsP3pG/nqj0PBtOTMKpwUYTWdWFbm2JrT+LMz8f6+l5yUazrJ22wMQd1EsBIVX9m9/4HAEBV/8O03+EclF++x6zrnq9x++pm4+XvdV8Xed+Vwmcd0/YC3S5+XU+E7zllF3zOQX0DwFtF5D4R+REAvwzgsxaOS5b4rqbspdpzm1TxrqwzqqPJmqS2Q3QtaxuWDQn3ne855dBaByhVfQ3ArwH4PIBrAD6lqt9ue1yimbiOyR5XSSSz/kZz/n4ukgFSD3i+55RjYGWhrqo+A+AZG8eidrqQgupEbBPzZX+UkPNIZQEjUCUIMxkAsLPAvAtFj6Ofs3WAlSQ6pAsfQitS/SoZuldoXuECBnTbyQAuAl4oPirQxIS1+DqijymoUzUtUGq7uKmLi7yv1O06wXLa69aCGaRyTS/Eqey/VpXvOeWQuN1Gh5hBKZfqh3CuJk9o3nvdV9ZaiOw42/UHzXY2fXPNeK4u3stdyH7rCm630UM2v3VGr22PwfYOt7My1myep6k8czEWM/5+LpIB+pb91hUMUB0S5EMYas+faaniVdme77F1PNevZ53AbmM4cVq52BnJINOSAZpuR9PH7LeuYJJERxQ/hOZEMOCwJ9Uk0yulagg+TatcAdgLgObrO+8NEfBvYTMZoI/Zb13BANURSX0IbV2EU9/Xqeyru88/1KyAGEFavs1kgL5lv3UFkyQ6xvs6qCYT/raSBOo8sWLvrM3vAnaCY90AFetYFHvEVFPVJAn2oDom+RTUw4ftX9TaXtjL2hOi5yYS50U/tvZQZzBJgtxpMrHv6sJfTD6oytVQV9PjRjKkaatsUOrlh8gtBqgIRfGhrZpNNutCG8nFFMDstkzLNGtaQHbW8eYdN4K5n3l/e1t18vq2+R7VxwAVGa8f2lkXoqqJDE0u4DFchGMVw3qlGX97WxVLWPmEKlFV77fl5WWl/cZj1eEw+4o9HJbft6rad/39N1XVwaD6z9Y9d1V12mDjfLbbH/L4Lc5tvi/zW5P3p63jUHoAbGqFWMEsvsiY3yRzzsoVtakZU/V3p72/bGarzWvLvPY2+Qy4zlwLmc1X4dyqdsoG2ToOpYWljhLVm3JFvoqeuuJ608OIX5/8S5SpSUUGW8eh7mKAikynPrSzLqZlF/h84aiLMj8RX/BLuQqALUspmT38NmWDbB2Huo3roCJS/NB6K1dUZlqVgSoX9KZXF5dlfrhWJ1PlNZ7xt7dVsSSpyicUDOegIuN108E28ygu5kiaHnPW78W4sDUkS3+34rRe/qt1q5iUHYfBqfuqzkExQEUoiQ+tiySBphfPmErtxNSWMo6SL2LYzTmJzw0BYJJE0pIoV+Q6SaBo1nyJ77ZMU2f9mMs2eN7+RDX8miYu+u0mzkFRemKqUGGKoV0BAmQ+f6SaBaV8znRlxc+8qRkggcm52+GQPamUsQdFdtj45h5rRp0tvjd1LOMom/HChXqP22RuaLi+nq2rMhONGJzSxQBFdtj45l4cqvNhXmB1OWQWosflYDhUFXjpJWBjY/LxjY3scR9/yt6sH+wZBijqt3mBNfScElXSqfWD9DoGKEpTDMNlBCB7+e+4I5tzMq2sZI/7mIPiot9uYoCieNWZF4mhRxNgDq148dVAFTNWV+s9btO0Rb/DIRf9po5ZfBSv4rxI7FeaafM4jtpduvbo0Vte1x4Bez2YjY39FVDyYOH6TzcaTWbr+TovucUeFNmRWq07nyy/NnleQ55afeZMmLVHuVh6MEmsH6RaWEmC0lGlbk5d8yo/RFYZwuw1AVlwMrPnQqZWs5IDVVW1kgSH+Kjf5gWZGMoT7SpbkFoUcliLPRiyjUN81A09GEosW5BaXHvErLVw9iWs8O/QGgMUpWPWXE5EPR2XyhakrqwAOztMrQ6JtQDdaBWgROSXROTbIjIWkbnjianjN6TAYikKG5AqcPLk/sfOngWOHGFqdQgxFMvtqrZzUM8C+EUAv2uhLVGLYTsB6jfVLCniypXs/spK9tjjj+/dv3gxG/ojf8xerVks13XCSh+SUlq9lVX1mqo+b6sxseI3JIqBWbFhZSWbf8qD04kTwKVL6QenVEcpfNcC7M2Qoqq2vgH4MoDjc37mNIBNAJtvectbNDXjsepwODm2NBxmjxP5NB5nN/O9mL8P838r/mwKVlcnP1P5Z251NWSrqvF5fTDPlZ+jeD92ADa1SmyZ+wPAl5AN5RVvDxs/MzdAmbfl5WUvL4Jt0y4KRD5NuxieP6964oTqyspeYFpZyR6L/SKf8kU3RNtT/8JsLUBVOkgPAlTqbwhqYTCY/MPnt8HAe1NmXQyXlvaatrKS3cz7sb9XU/6Mhej9pfyFmQHKopS/3ZEF5bmDux8f/6ZdDM+fnwxKKQWnXMoX3WJbXbY95WCu6ilAAfgFANsA/g7AXwP4fJXfSy1AqaY9Pk7arhfUJkA56n1NuxgWL/A2L/LmOcr+a+P4KV90fenCF+aqAapVmrmqPg3g6TbHSAWrJScu1MaDjs5bVlZId9PQi86cyTL82rxX82UWR44AL78MrK0B587t3W+73EJ1ck8nsyI6wM+aaVpxXqB7a+BYi68G1hqLSGRFXEPLg1Ne+ijfPHBjY++xpkFKjWUWS0vA1hbwla9k/83vD4ft1uGYF921tcmL7pEj/KwV9eULc+KrJqi3uBX7hHyN1IkTWXC6dCm7raxkj7XZ2dasAbi1lT1m/tfWgtTRKAtG587tXXzX1rIeWufW91jQhy/M3G6D0jTr01j2nq7786Y2vbU2520gP2R+2uL9tscuWwg8Hts7/rRhvpDbiJB93G6DyJaEhgxdfavOg0eZs2ftBI9QJYMoXhzio34IteNvB3YaNns2S0vZY+Z/bVZQ910yyLbiaxBggKpT2IOi/bqYgBCq3am+XgYzgeHIEeDUqcksvlOn7GWPlfXUbPXQXGNBafsYoGi/FBIQBoPpQbSNLgZnC8yssfy/5oXY9RwUEHeQMjMdgf3zZ7Zeo75hgKK0uA4gKQTnQMyU5rL/2jh+qut7OH/mBrP4aD9bmWcugonrrDjPWXe0X7G3kVLvo5jpaCvDsWuqZvExSYLcsdUbOXw4+5Tzk94Lqa7vmTZ/Zvt7TZ8SMRigKIw84IhkAajIDEocXqPIFefPxuPsvzYzHIEebVS4i3NQtJ+rBIRpys7FoEQJ8TF/1sdEDM5BkTt1Pi3F92HTonFtdTCLL+U5ndS4fq3NnlouxUQMzkFRv9jq3d26Vb6xRqLBqcmQUJ/mOGxzPX+W+kLmuhigyB0f1RISDyAumUNCeZDKv33fvFkeePo2x5EaX4kYsWCAInfM3ogLjgJgV3oQZhXy9fUs/XlW4VVV4KWXJgPamTOzAxr54ysRIyYMUBReWaCZVcPOYa+paz2IOkNCFy4Aly/v1dc7cCDbS+rYMe7JFINpiRjDYfwLmZtiFh+FVxZoAgzZ+cyS8pW4ULW2Xd57+vrX9x/jxRf3elBdvAimpC8bFb6uyr7wtm/Ly8tO9rmniJWnHmS3iIzHqsPhZPOGw+zx/N/bWl3df8zhMHvcJvO55Ocr3i/+/GOP7f/zLC2p7uzYbRv1G4BNrRArOMRHfiSy7UTZkFi+BbmN4b4miQtNNRkSKntsa2tvl1sinzjER34kkmVXNiS2vAxcvZpdpNsO9/kuKlp3SOjy5f2PLS1xDorCYA+K4mSWOppXFsmSYpbUzk52cd7aAm67zd7W477XslRZm5Nn7OVzUCsr2Q3Inj+z+CgE9qAoTgG2vSgbErt6NQtOORuBpGrigk8iwB13ACdOZLdLl/b+7cqV7N/YgyLfWOqI4hRw24t8SMxFWZliL62YKRg6Iyt/afM2FO8T2cBSR0QNFYOTzUWRsa9lKe5qwl1OKCQO8RGVcFmdundrWYga4hAfxSmSnW1ZCZzIPg7xUdoiWTeV6u6uRF3AIT6KUyLrpojIHfagiIgoSgxQREQUJQYoikOAyhFEFLdWAUpEPiIiz4nIt0TkaRFZsNUw6pkAlSOIKG5te1BfBHC/qr4TwHcBfKB9k4iIiFoGKFX9gqq+tnv3MoDF9k0iIrKvuHyOxW/jZ3MO6lEAfzLtH0XktIhsisjm9evXLZ6WiGi20WiyRJWNvb3IvbkBSkS+JCLPltweNn7mgwBeA/DJacdR1SdU9biqHj969Kid1hMRzeFzk0iya+5CXVV9z6x/F5FfAfBzAH5KQ9RNom4YDMoTIiLbcZfS43uTSLKnVS0+EXkIwBqAU6paedyOtfhLf9RDAAAFmUlEQVSIyDdV4IAxZjQeMziF4qsW338GMADwRRHZEpH/0vJ4RETWTdskkmM+cWtVi09Vf9xWQ4jaYNVxmmbWJpEAh/lixmKxlLzRKJvszi80+QVpYYFZWuR2by9yiwGKkmZmaAH7t1BnT4oAbhKZKgYoShoztKgq7u2VHu6oS53ADC2idHBHXeoNZmgRdRMDFCWtmKE1Hmf/NasGEFGaOAdFSWOGFlF3cQ6KOoHroIjSwTko6hVmaBF1DwMUERFFiQGKiIiixABFRERRYoAiIqIoMUAREVGUGKCIiChKDFBERBQlBigiIooSAxQREUWJAYqIiKLEAEVERFFigCIioigxQBF1THGDAu6JRaligCLqkNFocqPGfEPH0Shkq4iaYYAi6ghV4ObNyd2E892Gb95kT4rSwx11iTrC3E14fT27AZO7DROlhDvqEnWMKnDAGBsZjxmcKC7cUZeoh/JhPZM5J0WUEgYooo4w55yGw6znNBxOzkkRpYRzUEQdIQIsLEzOOeVzUgsLHOaj9HAOiqhjVCeDUfE+UWicgyLqqWIwYnCiVDFAERFRlFoFKBH59yLyLRHZEpEviMgbbTWMiIj6rW0P6iOq+k5VXQLwRwDOW2gTERFRuwClqreMu7cDYCIrERFZ0TrNXEQ+DODfAHgZwD+f8XOnAZzevft3IvJs23Mn7B8D+JvQjQio788f4GvA59/v5/+2Kj80N81cRL4E4FjJP31QVf+78XMfAHBIVVfnnlRks0qKYVfx+ff7+QN8Dfj8+fyrPP+5PShVfU/Fc/4BgD8GMDdAERERzdM2i++txt33AniuXXOIiIgybeeg/qOIvA3AGMD/BvBvK/7eEy3Pmzo+f+r7a8Dn32+Vnn+QUkdERETzsJIEERFFiQGKiIiiFCxA9b1Mkoh8RESe230NnhaRhdBt8klEfklEvi0iYxHpTbqtiDwkIs+LyAsi8luh2+ObiHxCRL7fx3WQIvJmEfmfInJt970/DN0m30TkkIh8XUT+Yvc1uDDz50PNQYnI4bwShYisAPgnqlo1ySJ5IvIvAfwPVX1NRP4TAKjqbwZuljci8g5kyTW/C+A3VLXz+6+IyG0AvgvgpwFsA/gGgEdU9TtBG+aRiDwI4BUAv6+q94duj08icg+Ae1T1myIyAHAVwM/37O8vAG5X1VdE5CCArwIYqurlsp8P1oPqe5kkVf2Cqr62e/cygMWQ7fFNVa+p6vOh2+HZAwBeUNW/VNW/B/CHAB4O3CavVPVPAfxt6HaEoKp/parf3P3/HwC4BuBNYVvll2Ze2b17cPc29dofdA5KRD4sIt8D8K/Q70KzjwL4k9CNIOfeBOB7xv1t9OwCRRkRuRfAuwBcCdsS/0TkNhHZAvB9AF9U1amvgdMAJSJfEpFnS24PA4CqflBV3wzgkwB+zWVbQpj3/Hd/5oMAXkP2GnRKleffM2VbB/Zq5IAAEfkxAJ8GcKYwktQLqrqzuwPGIoAHRGTqUG/rYrFzGtLrMknznr+I/AqAnwPwU9rBBWk1/v59sQ3gzcb9RQD/N1BbKIDdeZdPA/ikqv630O0JSVVvisiXATwEoDRpJmQWX6/LJInIQwB+E8B7VfX/hW4PefENAG8VkftE5EcA/DKAzwZuE3mymyDwJIBrqroWuj0hiMjRPGNZRH4UwHsw49ofMovv08hKrr9eJklV/0+QxgQgIi8AeAOAG7sPXe5ZFuMvAHgcwFEANwFsqerPhG2VeyLyswAuAbgNwCdU9cOBm+SViDwF4J8h227irwGsquqTQRvliYj8JIA/A/C/kF33AOC3VfWZcK3yS0TeCeD3kL3/DwD4lKr+u6k/38GRJSIi6gBWkiAioigxQBERUZQYoIiIKEoMUEREFCUGKCIiihIDFBERRYkBioiIovQPHRS5kh+nE8IAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_pred = np.squeeze(y_pred)\n",
    "\n",
    "plt.scatter(X_xor[y_pred == 1, 0], X_xor[y_pred == 1, 1], c='b', marker='x', label='1')\n",
    "plt.scatter(X_xor[y_pred == 0, 0], X_xor[y_pred == 0, 1], c='r', marker='s', label='-1')\n",
    "\n",
    "plt.xlim([-3, 3])\n",
    "plt.ylim([-3, 3])\n",
    "plt.legend(loc='best')\n",
    "plt.tight_layout()\n",
    "# plt.savefig('./figures/xor.png', dpi=300)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now let's try 2 -> 3 -> 3 -> 1 NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# input & output\n",
    "X = tf.placeholder(tf.float32, [None, 2])\n",
    "Y = tf.placeholder(tf.float32, [None, 1])\n",
    "\n",
    "# 1st layer\n",
    "W1 = tf.Variable(tf.random_normal([2, 3]), name='weight1')\n",
    "b1 = tf.Variable(tf.random_normal([3]), name='bias1')\n",
    "layer1 = tf.sigmoid(tf.matmul(X, W1) + b1)\n",
    "\n",
    "# 2st layer\n",
    "W2 = tf.Variable(tf.random_normal([3, 3]), name='weight2')\n",
    "b2 = tf.Variable(tf.random_normal([3]), name='bias2')\n",
    "layer2 = tf.sigmoid(tf.matmul(layer1, W2) + b2)\n",
    "\n",
    "# 3nd layer\n",
    "W3 = tf.Variable(tf.random_normal([3, 1]), name='weight3')\n",
    "b3 = tf.Variable(tf.random_normal([1]), name='bias3')\n",
    "hypothesis = tf.sigmoid(tf.matmul(layer2, W3) + b3)\n",
    "\n",
    "# cost/loss function\n",
    "cost = -tf.reduce_mean(Y*tf.log(hypothesis) + (1 - Y)*tf.log(1 - hypothesis))\n",
    "train = tf.train.GradientDescentOptimizer(learning_rate=0.1).minimize(cost)\n",
    "\n",
    "# Accuracy computation\n",
    "# True if hypothesis>0.5 else False\n",
    "predicted = tf.cast(hypothesis > 0.5, dtype=tf.float32)\n",
    "accuracy = tf.reduce_mean(tf.cast(tf.equal(predicted, Y), dtype=tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 1.0599269\n",
      "4000 0.6059752\n",
      "8000 0.18207191\n",
      "12000 0.062594675\n",
      "16000 0.0369588\n",
      "20000 0.024818217\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "# Launch graph\n",
    "with tf.Session() as sess:\n",
    "    # Initialize TensorFlow variables\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    for step in range(20001):\n",
    "        sess.run(train, feed_dict={X: x_data, Y: y_data})\n",
    "        if step % 4000 == 0:\n",
    "            print(step, sess.run(cost, feed_dict={X: x_data, Y: y_data}))\n",
    "            \n",
    "    print( sess.run(accuracy, feed_dict={X: x_data, Y: y_data}) )\n",
    "    y_pred = sess.run(predicted, feed_dict={X: x_data, Y: y_data})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X9sXtd5H/DvQ0+NCpeUbEeOnDCAPbRI1hkJAzFihQ32tnqrERRxW6BAjWFr4AHCgEx8SbVAmxmQqC3BNiSlSHoBVgMO1gKpiwKZs6Br0yTYki6oxZQK6Myp7cALMIRLnaiqJdlY3dp6n/1xefXe9/L+Pufce8693w/wgrovX9573lfSfXjOec5zRFVBRETkm5muG0BERJSFAYqIiLzEAEVERF5igCIiIi8xQBERkZcYoIiIyEvGAUpEDovIN0TkORH5tohcsNEwIiIaNjFdByUiAuB2VX1dRA4B+DqAkapestFAIiIapr9legKNItzr+4eH9h9c/UtEREaMAxQAiMhtAC4D+HEAn1bV7YzXnAZwGgBuv/32E+9973ttXJqIiAJz+fLlv1DVY2WvMx7imzqZyFEAzwA4o6rP571ucXFRd3Z2rF2XiIjCISKXVXWx7HVWs/hU9RqArwJ42OZ5iYhoeGxk8R3b7zlBRH4UwEMAXjQ9LxERDZuNOah7APzW/jzUDIDfU9Xft3BeIiIaMBtZfN8C8AELbSEiohxvvvkm9vb28MYbb3TdlMoOHz6M+fl5HDp0qNHPW8niIyIit/b29jA7O4t7770X0fJTv6kqrl69ir29Pdx3332NzsFSR0REAXjjjTdw1113BRGcAEBEcNdddxn1+BigiIgCEUpwipm2lwGKiIi8xABFRESVPPbYY7j77rtx//33t3I9Bigioh5KFwmyUTToIx/5CL74xS+an6giBigiop5ZWwNWVydBSTU6XlszO+8DDzyAO++807R5lTFAERH1iCpw7RqwuTkJUqur0fG1a3Z6Um3hOigioh4RAS5ejP68uRk9AGA0ip4PKRGQPSgiop5JBqlYaMEJYIAiIuqdeFgvKTknFQoGKCKiHknOOY1GwHgcfU3OSTX16KOP4tSpU3jppZcwPz+Pp556yl7DM3AOioioR0SAo0en55zi4b6jR82G+Z5++mk7jayIAYqIqGfW1qKeUhyM4iDFOSgiIupcOhiFFpwABigiIvIUAxQREXmJAYqIiLzEAEVERF5igCIiotpefPFFnDp1Cm9729vwqU99ysk1mGZORES13Xnnndja2sLnP/95Z9dgD4rIkIt9d4iMzM1FeeXpx9yctUvcfffd+OAHP4hDhw5ZO2caAxSRAVf77vRKCzdLSnnttXrPe4oBiqihPu2741RPbpbUPgYooobi8jFxIc6ZmUmBzhDLyhCV+fSnP42FhQUsLCzg+9//vvPrMUARGejLvjtEVXz0ox/F7u4udnd38c53vtP59RigiAz0Zd8dorpeeeUVzM/PY319HR//+McxPz+PGzduWL0GAxRRQy733SEyMjtb7/kGjh8/jr29Pdy4cQPXrl3D3t4e5iwnvnAdFFFDLvfd6ZXZ2eyECIs3S0qx3JPpCgNUC5L7smQdU7j6su+OUz25WVL7OMTnGNfJ9F8f9t2hMGhg48am7WWAcojrZIjIlsOHD+Pq1avBBClVxdWrV3H48OHG5zAe4hORdwP4bQDHAYwBPKmqm6bn7YPknMTmZvQAuE6GiOqbn5/H3t4erly50nVTKjt8+DDm5+cb/7yYRmMRuQfAPar6TRGZBXAZwM+p6p/l/czi4qLu7OwYXTckqtEizth4bB6cOK9FRKESkcuqulj2OuMhPlX9c1X95v6fXwPwAoB3mZ63L1ysk/FpXiu0QqmhtZdoyKzOQYnIvQA+AGA743unRWRHRHZC6qKacLFOxqd5LReB0mUA8SmwE1E5awFKRH4MwOcArKjqgbxSVX1SVRdVdfHYsWO2Luu1vHUyo1HzdTK+1H9zEShdBhCfAjsRVaSqxg8AhwD8EYCzVV5/4sQJHZLxuPi46Tmj22r0sHHOJm0YjabbMRo1a0vyXPE50scu2ru8PH3uLj5HasHs7PRffPyYne26ZYMEYEcrxArjHpSICICnALygquum5+sj2+tkfKn/ZrNQahs9w6z2JnHIr8e45UeQbAzx/T0A/wzAPxKR3f3HhyyclzL4VP/NdqB0XRk8q71bW8DKCof8iHxkI4vv66oqqvo+VV3Yf/yBjcbRQS7mtZpwlQDiqmeY1d7l5eh7W1vcy4nIR6zFFyAf6r/ZLpSaDiAXL06OAfP3l9XejY3oe1tbk9c5/Rzn5vKLprJeHdEBDFCB8qH+m81A2UZl8HR7s6yuOgxSnAchqoUBiozYDJRt9Azjc8U9tq0tNz22+BocKvQEt/wIEgMUeaWtnqHrHtvaWpRsEZ9bFWCs6hCHUIPEAEWFfK/5Z9I+Vz225KJgYNI72zA7LdHgcLsNyuV7aSAb7XPRY8tb00VE9TBAUSbfSwP53r6sNV2aN9/BeRCiTBzio0y+72Xle/sy13Q9dsOLthGFwng/qCaGth9UyFzsZWWTj+0rWtPlSwAl6lJr+0FRf/lS8y8pee3xOCpTlNR1+wB/qn0QhY5DfJTJdWWHJpKp22trwBe+AOzuAktLwJ/8CXDiRLftS7e162ofRKFjgKJMbVR2qCOZFBH3kHZ3o68nT0bBc3cXWFgAjhzxIxD4UO2DKGScg6JCPq2DSvbqsoxGwPr69JwUEfmHc1BkhU+9gLL9nC5eZHCiCubmon9M6cfcXFjXGAD+d6ZgZCVtJPmQIGGMNzb32ijay8LAVnAOioKQHN5L7uMETI59SZAwwhsb0S0MUBSEdNLGhQuTwHTHHcD585PXBBuciGgKkyQoKMkkjfifbvI4+OBU9AaCH7/0RBufMf8eCzFJgnop+f8+np7J+h5RsDgPeQsDFFGIeBNrro2ivSbX4DzkLZyDIvJJ1Z1f27iJzc3ltyXkDQDbaHvIn49HGKCIfOLTjY2/yVPHOMRHhIPz1pzHJuoeAxQ5FcKN3/edg4mGigGKnAnhxu/7zryE4SWEcOflWzgHRU4kb/zAwU37fFmz5PvOvLmqJlP0wdDmwnyah+wYF+qSM1nVx7Nu/D5UTLexM68P78OqOll8LjP+uOi1d7hQlzqXVX08HZx8GAa0sXOwD+/Duhs3ojeSfmQFnKH1cqgVDFDkTNmN34f5n/TOweNx9DXZpirn6Pp9EPUR56DICVVgZSWqOB4P662sHKw43vX8j42dg0WiXXwXFqbfh0+7+wYjb6iQBokBipy4cAHY3o4qjieH+ZaWpm/8cUBIzlO1nZywtjY9XxS3qWobVIHr1ydb0Md2d4EHHujBXFSb6gSnPiaE0BQO8ZF18ZDX9vbkudXVqDe1tBRtjZF8ren8jw0mOweLRFvNHz8+/fzCQvQ+Llwwbx+h2lwY9YqVACUinxGRH4rI8zbOR/6ps+A27oGMRlFQmpmZzPFsbExvj2E6/+OD+H288sr087u7wBNPAK++Gs57aYxrd8gBWz2o/wzgYUvnIs80yVCrksGXN/8zGoW18WD8PhYWDn7v5MnpoFxbKItU62T8EVVkJUCp6h8D+Esb5yK/NM1Qqzp0t7Y2HbjiIFWWnu1TCaXkHFS8y2/sp37K8ORM36Y6QvmFpqLW5qBE5LSI7IjIzpUrV9q6LBlK9mo2N6eH6+LAkg4O43G9obu68z++rTmKe1Dp4ARMz8N5zZcbG4cKzfTsF5rWApSqPqmqi6q6eOzYsbYuSxYUDddlBYuzZ6PehIuhO1/WHKWvc+5c9DVOqx+Po4C1vR3IfJovN7ZQhwp9CfA9wzRzKpU3XLe+Xlxvb329eep2Hh/WTq2tRe872YM8exZ47rnpdsRzTyHNp1FDvgT4vlFVKw8A9wJ4vsprT5w4oRSG8Vh1NIp+jR2NouPl5cnxzZuT4/gRv851u5LXdH295HXTn0fy+ObNg683kt2fiB42tXWdLszOZr+v2Vl71/Dl8/OlHSUA7GiFWGErzfxpAM8CeI+I7InIv7BxXupeOtMuXtOzvJzfM3Ddk+li7VR87rgHt7ycPSc3k/ofZfw5+DYnE+JQFns3wbKVxfeoqt6jqodUdV5Vn7JxXvJDnGkXz/9sbUXPnzs3KWeUtLLiLlh0sXYqPc+WxVlQ9m1Ohjd7v/n2C40hzkFRJRcuRMFpfT063tycDkxnzkTPxQFrext49ln7N20btfPqSCZlAJOagumgvLrq+f5RZYa0v1Sf+Z5MUhMDFJVK36TX16dr5wFh1c6rIy8pA4iG+TY2pve8CjZI9ezG1joGeCe4YSFVkhxaS1tYmC6UGt+4g7xR51Cdnl9Kvsf4szl6NPD9n6oIcfPAENvcc9ywkKyKC6Im3bwZDbWlq3ing1NbVR9cXScrKSOpavULZ0JMXGhTz+ZlhqT3Acqnkjghi9f6JJ09C/zGbxysQZdMKGir6oOr6+QlZWxtTV+v095iUeKCSdDKCnx5fL7Z+5ZoQpX1OkD5VhInVOmbdNxz2twEFhcnVSPSGXXjcTtVH5JzZLav05eCtrfUybYreq3Nmz17gJSnymIp2w/bC3XTiyHH4/IFlW0t7OxK1mdi4vz56c8t/jwffDD7+fPnp49dL+Stc50mn43tz7NQ3YWlRYszTRZscpEwOYKKC3WDT5LIKjsTT1ifP39wYr/NkjhdKfpMTHqPycy55HHe88njZILBeOzm869yHVefjVVlk/qm26JX/T/fVnIBkxgGZxBJEmVDO0D5nkR943q4K+u4qBp5VoKBiwW1Va7j8rMxkh7iKsNFsTQUVbpZth82h/iKhnbaGl7yjS/v2+Ywa9EQW53r+PLZTKk7PFd3SI9DfOQZDGWID8ge2gGmJ/bTlbaH0JNqY1itjI0htSrnqHMdXz6bW+pcPD1+mvV9IH8YcHa2ekKDjXNUwSG+wak6xNfrHlTexH48gd9XvvUSTBIM6vaOyq6T99mkq5C3ymYPKkRtVBsnr6BiDyroAFXl5tVq9lXH0sOay8v9yF60FXCT51lYiIJS8vjcOTftz5R3Uy56xDfsvgUoGpyqASroWnxNCoeGPKynmp8tlxziSm4/fuHC9GdSdk4fxX+vyWzMJkO0IsCRI5PSTGfPRtUxvva16PjBB1v8PKomOmjGEBfrvtFQVIlith9trIPqm6LhyrJNBeOeVahDnqY9qPTr3nrL7RBo3rq8KXV6TEQ9gzY3LOxaUZpzH6hWS6dPbqK3tXUwGcTLFOt96evHx8l2Ntn/KauayK/8StSTSrKVNJN1vVOnokfyuUJxiOqyFA+rO5APqkQx2w9u+V5fVi8inkeJv5/eej3rN3mfkidiZT27pj2/ojnKhYXpzyGer0v+bF1FPdn0nGCjuac2cZ4rW92EDiaAZMIQkiSGZjw++O8860ZYFHzS5+g6OFXN0ms6jJsX2JPnX1qaDiAmQ59Z1zt58mBArBWgurihMUBlq/u58HPMxADVM0U32qyeQN5N3scelOt2pYPyuXPTc3NxcF9aspP1mL5e1i8P19Egi6+uKr+9N8kmtHXtEDFAWcEA1SNFvYys4JT8mayirT4Wz3XVs6uy7imrB2oSnNLXO3Pm4C8TyeHZW2wHqCrnqhucqrSjLOhV5WOQY4CyggGqZ9LzMDdv1p9HMZnLKTo25aoHVXeRr2mALJuDSj/+6lDN3ktdXQUoW+/Dx5s7A5QVDFA9lDVUV7c3VDfYuE5Nd92zq9J+mwEy63onT6oePz59/oUFtR8Y0lwEqCq9Fwao5q8fCAaonmtjTVNbw4JtBMG8YxfvMX3+9Jq0+PzeB6im+hygmMVnRdUA1YtisaFSLd5HyfXPV71GG3tqtfFe8rjeIyrv/BubNd9g3SKtVYqwuijUWvYXV/W8LCLbW1WLxTJAdSSIjfP2qXpW/dsB1wEy8/wzDS5Q5/9rlWrkLiqWM0BRiUFsWBgqVb+rOiTFbUtyseFg1+J7YTwGkz62cf7keVoJ8DduZA+QJQNPldfUVVQTsE69wLzXuqw5yAoaXmGA6kBc/DQu2TMz4+c+VcnA2aTMUGjW1qKSRCsrk/v0ykr0nGmvNqsE0huHelrcNS/o1Q18LoJnmbwivtzFuBNBVzMPma0K3S41qRYfKlXg1VeB7e3oEdvair4uLVWbtsk7d9xjBhKbZ755I/uXEt8+2LxhwJjtDQyJ9nEOqiNtJR/Y0GUCQ1vW1qIABUyCUuzMmcnfU9N5wlp/3y7nXprMOVX5y+5Ld5rzXq3gHJTH2h46S5+v7vnrVouvez3T9pmKezjpwJRmMk+Y7H3Gcn8ZcTn30uYQFudzyBCH+DrQ5tBZ29mCda9n2r6mvbvk60SijQtVs4PUE09EDyDa7PDIkWbDfFnJJplBqi/DZZzPIUPsQXVkbW365hQHKZtBo+1swbrXM21fVuLB6mr5Z5j3c5cuTb/u/e8/+LO7u8D16/U+u6ElmwSti8xByldlNW/ZA8DDAF4C8DKAXy97PStJtKftCuZ1r9e0fU0rQBT93DveMd2OM2dU3/726ecyi7xW4M1uxvn5dQdfW6fSuem1aFDQVqkjALcB+N8A/jaAHwHwHICfLPoZBqh2tb0HVN3rNW2fjeCWDDxAVJbo5s3iIq82SiBlHbeiTtCoGpzyyvYwQDnhxb8jQ1UDlI0hvpMAXlbV76rq3wD4XQCPWDgvWaA5cx/qaFip7vVM2peZePCZuahCQ8HEfNbPffjD0bDbxka0Lm1jI0otf8c7pl/XdA4qvm7RcStsDWElw01f5swC0HRYO1Q2AtS7AHwvcby3/9wUETktIjsisnPlyhULl6Uy8T/eNrMF61zPtH1ZwU0qTMxn/dz161GiRDJoLC0BP/jBdNuazEHZlvU5Vtbm4lfO51ilGk4FGltsZPFl/R544KNS1ScBPAlE66AsXJdKtL3Qtu71TNqXDm7x4lds5v9M0c8lF9HGna477jD87BzUuTPJelRteT0be1ZWJf8Nbm5O/s36un7SBuOFuiJyCsCaqv7M/vHHAEBV/13ez3ChbrvavjHVvV7T9mXdrAsLsO7/W69zkzf67Cwv+iwKrmU3qUaBzUL745claxuWnZqKqYZfvLnqQt3aSRHpB6Je2HcB3IdJksTfLfoZJkmQsToZZhkT861MNDtIEmiSGJLOXLx58+BxJsO9jM6fV11amuz0HO+NtbRULXuxD8kAtrWdlesK2tywEMCHAHwHUTbf42WvZ4AiY3WDUxeZYwVtMbn5Nsl6zLuxxcEqK2CYpMant7tfXj54XNRuF2n5oQe8tjYQbUOrAarugwGKjNUNTl3sYFrQHpMbf9PfoG/enP65ZDp9fI7kw/RmmA5SVYOTixuxN+vQDPXlfTBADdGQtpf2qaeUp6CNTW6+JjfurMAWP97//ihYpYfg6gTDvN5Jurdn2uNruv6sLz0P1fB7gqoMUMMUwk3bFhc9KNsBPud849nZxjffJr9BZ81BZfVqsobgqgwn5rXp3LlmPahku+sGtiqfgWnAI3MMUEPEAGX2GbT4+ZncfJv8Bh0HkWSCRN4jGZzKbupFvZO4OkdRACx6j7YDStsVVSgfA9QQDSlAGWbx1f55y+35q0OzuTdfV0M4Wdl7WT2cunNQecHk3LlmWXwuhuTYg/JL1QDF7TbIjIPFqJWYbK4H2N/yoeb5Dr/5WuZC4SNHokoVLrZHmZmZXhidZ2UlKvVUdRF10e7QcZvj129sTB9nsb3APP4MyxZnk4eqRDHbD/agHOmiB9Xkmi6TOaq2x6T3Vfe6OY+sOZs2JvOTvZmyIbgqvTlXvRObPcm+ZL/1BdiDGqDZ2fzejE9sbWSX13sLRHovsLjHBLgtZROf5447onqDS0uTng0AbG9H38u6Xvo5l70Tm8V119aitmZ95uQv41JHTbDUUY80KYdjqwRQnbtLesjR5GcBO8Ex572qZpeySd5gbYmbYFKKqO1dmyl8VUsdsQdFfpmbsz93ZfpLWFZ7HPXc4pt70upqVGn97Fn7N30bvRT2TsgVbvlO7iT2YKrM1ZDd3Nz0/lBVuRoezThvcrhsYSF6bmEhOj5xot1tFdLXKDtOq/oR1z0vDQsDFGVL39AzNv4DUHwD92l+qKgtRakMTXpzVVIkMs6bzF67fHmy/xQQfW1rW4WyTfFsbZo3tM33qD4GKN9UDQyur1U1kaHJDdy3pA2PrK1FQWhmJmO34BaCk2rxpnjjsZ1N88quw54UAWCauXfaTBVvkBp9qx1VF6a6fJ912mD7c3X899TlwtKya9tqGxfPDhcqppkzi883lje5a3ytIlojnSyvzTbfZ1lbytrb5HN1uEA52ZuouzGhLZqTSVj1+7auQ/1UNYuPQ3zUjbxhvlCG/27csDdnlZJXSWE0alZJIS0dj7OOszIJ49eVfb9OO2ych3qsSjfL9oNDfAVCGeKr8rq6lSGaVpgwGZL0eCsSFzX5yioqlNXBS9fza1rtom9bYFA9YCUJMmJSlaLpr8C2KkxkcVkX0BGblRSA6K8lTkwADg4dxiOhRXXw0vX8mtbJs11vj/qJc1C+abP4qsm1XMyVNT1n0c+5LlobmOT8VixrXis9bVflGCh+TV576v4MhY9zUKFyOLdh9Vo+zSEVtaXt4NTmMoEcRXNMyZ5KLCvpou7xhQvN1jTZ7CUWvW8KEwMUNdNmIAWKb/ZttyVPnfVjjpQtfo2Pk0wTE5JDh12taeKi335igKLw+FShIqnjdpUFivF4es5pPI6+Jl/fRNwrW16OzjUzE31dXvZjcTF7UuFikgTZYWPuLC8xoy/Sd2rLQ5DJ4bu87TpcJSZcuJD/vOteTJX3TWFikgTZ4VPSRB1lgbVO4G26UtUyLVn8ajsxQTXahXdr6+D3lpejvabaCBJl75v8wSQJoirK5ow6nlOqq8ock+30dR+4mFuj7jFAUZg6zJLzVXLuxeYcUxmRaAfe5eXp55eX83fmtamr903ucQ6K/FVnTsqHHk0Hc2jJ4TkR4MiRbha/nj8fDfNlPe8aF/32FwMU+cvGHE+b8hIeHLU7a6v169ejINXm7rZxD2Zr62Bx27Z21+Wuvv3EIT6yw6eFu76x/NnEy7zi1OqVlelhruvXi+ecbHNd3LZOO4qOKTzM4qNwVKmbU5fNLL4WJHtNwMHsuS5Tq1m2iKqqmsXHIT4atrIg41Edv6xir2ldDmuxB0O2cYiP+mEAQ4nJobO4YkN67RGz1rrDWoD2MUBROHwqCtuRrGKvy8vAzZtMre4SawG6YRSgROQXReTbIjIWkdLxRCIjvhSF7ZAqcOrUwedWVycp5kytbhdrAbpjOgf1PIBfAPCbFtriN88my2l44pJC29vR8fJy9NwTT0yOL16cLvdD7nVVC3AISSlWsvhE5KsAflVVK6XmBZnF10ZdOKISa2vAq69Gf07OPy0tAc8+G/4NKuSbbpu1ALPWwK2uRr3nEIYVvavFJyKnRWRHRHauXLnS1mWJemVtLSq+urEx/XwcnOJRz1j62Gchz+O0WQtwSEOKpQFKRL4iIs9nPB6pcyFVfVJVF1V18dixY81bTESZN8Pz56P5qXjhbjwkeOqU/zf5kG+6bdcCzMrmjK/dt+oZpXNQqvpQGw0h8pZH84/pm2GyrNDCArC7O5mjAibDgEtLfg+XhbynUxe1AONrxJ8T4P/n1ATnoKriHNRwefZ3nzf/cORI9Hx6bVSbezKZCnlPpzbnz5K/qMRCCOaxqnNQUNXGDwA/D2APwF8D+AGAP6rycydOnNDgzM5mJThHz5P/TP7+spPbo4fL6xYYj7OPx+ODl0q/1vSaeV9tnH80mm77aGTv/H2R/Jzizyd97DsAO1ohVhilmavqMwCeMTlHMJhKHrauNh50dN2sskLxnFPayop5DyrutR05EhWjXV8Hzp6dHJtmjxUNXQLh9AzaMKTtRViLj8Lk0byQD+LgFA/vxZsHbm1NnmsapJIJDPE819e+Fn2Nj0cjsyGt5E13fX36ppvcPoQiQ9lehEv6KEyBbcXuWryr7dLSZM5pYyP689KS2c62yayx3d3oueRXW3Mfa2tRMDp7dnLzXV+Pemi+ZyF2YQjFebndBoWpbuKCSaKDSW+t5QSL+JTxZdPHpufOqlJhK5GhaJgvpAQAKsftNohsCWjI0NVv1VkLUWOrq3aCR8ip5uQGh/hoGLra8bcHOw0nezYLC9Fzya82F6RmVWsPKTilPwOuQDHDHhQd1McEhK7aHernlZBMYDhyBHjwweksvgcftJc9llcyKIQgFXp9PB8xQNFBISQgzM7mB1ETfQzOFiSzxuKvyRux6zkowO8glcx0BA7On/lcxcNnDFAUFtcBJITg3JFkSnPWVxvnD3V9D+fP3GAWHx1kK/PMRTBxnRXnWVmjIWqzZJBtIZdqapN3223QANnqjczNRf/L+T99EEJd39PWlhtDSsRggKJuxAFHJApAacmgxOE18lxbW26EvGdWE5yDooNcJSDkyboWgxIFpI35syEmYnAOityp878l/e+wadE4Uz3M4gt5Tic0rj/r0LfZiHEOiobFVu/uxo3sjTUCDU5NhoSGNMdhm+v5s9AXMtfFAEXutFEtIfAA4lJySKjqNupDm+MITVuJGL5ggCJ3kr0RFwIqF9SFZBXyzc0o/bmo8Koq8Oqr0wFtZaU4oFF72krE8AkDFHUvK9AU1bBz2WtKZg+WZRoGoM6Q0IULwKVLk/p6MzPRXlLHj3NPJh/kJWKMRv4vZG6KWXzUvaxA09WQXUuVJNpKXKha2y7uPX3jGwfP8corkx5UH2+CIRnKRoUx9qCIWtbWPE+dISGRaIPDM2cOnmdhod83wdCEupC5CQYoakcPtp2wMdzXJHHBpLl1h4SyntvdnexyS9QmDvFRO/qUZWcw3Nd2UdG6Q0KXLh18bmGBc1DUDfagyE89S1ZIanstS5UhoThjL56DWl6OHkDUg2IWH3WBPSjyU1fbXuSVebLIx035RIA77gCWlqLHxsbke9vb0ffYg6K2sdQR+cmHbS8ctKFoUz4fStbEbytuQ/rrVePDAAAGu0lEQVSYyIaqpY7YgyJqke+b8g0pQ4z8xx4U+cmHHpTDwrEs4EpDxh4UkSmHmYfsqRCVYxYf+akP66aIyAh7UOSnPq2bIqJG2IMiIiIvMUAREZGXGKDIDz2uHEFEzRgFKBH5pIi8KCLfEpFnROSorYbRwHRVOYKIvGXag/oygPtV9X0AvgPgY+ZNIiIiMgxQqvolVX1r//ASgHnzJhER2Zde383it/6zOQf1GIA/zPumiJwWkR0R2bly5YrFyxIRFWtrk0iyqzRAichXROT5jMcjidc8DuAtAJ/NO4+qPqmqi6q6eOzYMTutJyIq0eYmkWRX6UJdVX2o6Psi8ssAfhbAT2sXhf2oH/K2uWDlCDLU9iaRZI9RsVgReRjAOoAHVbXyuB2LxRJR21SBmcSY0XjM4NSVqsViTeeg/iOAWQBfFpFdEflPhucjIrIub5NIjvn4zagWn6r+uK2GEBG5ULRJJMBhPp+xWCyFz+G+TRQ+3zeJpHzcsJDC58PmhuQ9bhLpj7bmoIiIgsBNIsPDAEVERF5igCIiIi8xQBERkZcYoCh8edUmWIWCKGhMM6fwMZWcqJfYgyIiIi8xQBERkZcYoIiIyEsMUERE5CUGKCIi8hIDFBEReYkBioiIvMQARUREXmKAIiIiLzFAERGRlxigiIjISwxQRETkJQYoIiLyEgMUUc+oFh8ThYIBiqhH1taA1dVJUFKNjtfWumwVUTMMUEQ9oQpcuwZsbk6C1OpqdHztGntSFB5uWEjUEyLAxYvRnzc3owcAjEbR8yLdtY2oCdEOfq1aXFzUnZ2d1q9LNASqwExibGQ8ZnAiv4jIZVVdLHsdh/iIeiQe1ktKzkkRhYQBiqgnknNOo1HUcxqNpuekiELCOSiinhABjh6dnnOK56SOHuUwH4WHc1BEPaM6HYzSx0Rd4xwU0UClgxGDE4WKAYqIiLxkFKBE5N+KyLdEZFdEviQi77TVMCIiGjbTHtQnVfV9qroA4PcBnLPQJiIiIrMApao3Eoe3A2AiKxERWWGcZi4inwDwzwFcB/APC153GsDp/cO/FpHnTa8dsLcD+IuuG9Ghob9/gJ8B3/+w3/97qryoNM1cRL4C4HjGtx5X1f+aeN3HABxW1fOlFxXZqZJi2Fd8/8N+/wA/A75/vv8q77+0B6WqD1W85u8A+G8ASgMUERFRGdMsvp9IHH4YwItmzSEiIoqYzkH9exF5D4AxgP8D4F9W/LknDa8bOr5/GvpnwPc/bJXefyeljoiIiMqwkgQREXmJAYqIiLzUWYAaepkkEfmkiLy4/xk8IyJHu25Tm0TkF0Xk2yIyFpHBpNuKyMMi8pKIvCwiv951e9omIp8RkR8OcR2kiLxbRP6HiLyw/29/1HWb2iYih0XkGyLy3P5ncKHw9V3NQYnIXFyJQkSWAfykqlZNsgieiPwTAP9dVd8Skf8AAKr6ax03qzUi8ncQJdf8JoBfVdXe778iIrcB+A6AfwxgD8CfAnhUVf+s04a1SEQeAPA6gN9W1fu7bk+bROQeAPeo6jdFZBbAZQA/N7C/fwFwu6q+LiKHAHwdwEhVL2W9vrMe1NDLJKnql1T1rf3DSwDmu2xP21T1BVV9qet2tOwkgJdV9buq+jcAfhfAIx23qVWq+scA/rLrdnRBVf9cVb+5/+fXALwA4F3dtqpdGnl9//DQ/iP33t/pHJSIfEJEvgfgn2LYhWYfA/CHXTeCnHsXgO8ljvcwsBsURUTkXgAfALDdbUvaJyK3icgugB8C+LKq5n4GTgOUiHxFRJ7PeDwCAKr6uKq+G8BnAfwrl23pQtn733/N4wDeQvQZ9EqV9z8wWVsHDmrkgAAR+TEAnwOwkhpJGgRVvbm/A8Y8gJMikjvUa1wstqQhgy6TVPb+ReSXAfwsgJ/WHi5Iq/H3PxR7AN6dOJ4H8P2O2kId2J93+RyAz6rqf+m6PV1S1Wsi8lUADwPITJrpMotv0GWSRORhAL8G4MOq+v+6bg+14k8B/ISI3CciPwLglwB8oeM2UUv2EwSeAvCCqq533Z4uiMixOGNZRH4UwEMouPd3mcX3OUQl12+VSVLV/9tJYzogIi8DeBuAq/tPXRpYFuPPA3gCwDEA1wDsqurPdNsq90TkQwA2ANwG4DOq+omOm9QqEXkawD9AtN3EDwCcV9WnOm1US0Tk7wP4nwD+F6L7HgD8a1X9g+5a1S4ReR+A30L0738GwO+p6r/JfX0PR5aIiKgHWEmCiIi8xABFREReYoAiIiIvMUAREZGXGKCIiMhLDFBEROQlBigiIvLS/wfTsHpUDS+UswAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_pred = np.squeeze(y_pred)\n",
    "plt.scatter(X_xor[y_pred == 1, 0], X_xor[y_pred == 1, 1], c='b', marker='x', label='1')\n",
    "plt.scatter(X_xor[y_pred == 0, 0], X_xor[y_pred == 0, 1], c='r', marker='s', label='-1')\n",
    "\n",
    "plt.xlim([-3, 3])\n",
    "plt.ylim([-3, 3])\n",
    "plt.legend(loc='best')\n",
    "plt.tight_layout()\n",
    "# plt.savefig('./figures/xor.png', dpi=300)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Keras Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.contrib.keras as keras\n",
    "# If error, make update `dask` package in conda\n",
    "# conda upgrade dask\n",
    "# conda install tornado=4.5.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential()\n",
    "\n",
    "model.add(\n",
    "    keras.layers.Dense(\n",
    "        input_dim=2,\n",
    "        units=3,  \n",
    "        kernel_initializer='glorot_uniform',\n",
    "        bias_initializer='zeros',\n",
    "        activation='tanh'))\n",
    "\n",
    "model.add(keras.layers.Dense(input_dim=3,units=3,activation='tanh'))\n",
    "model.add(keras.layers.Dense(input_dim=3,units=1,activation='sigmoid'))\n",
    "\n",
    "#adam_optimizer = keras.optimizers.Adam(lr=0.1)\n",
    "#model.compile(optimizer=adam_optimizer, loss='binary_crossentropy')\n",
    "model.compile(optimizer='adam', loss='binary_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = model.fit(x_data, np.squeeze(y_data),\n",
    "                    batch_size=10, epochs=200,\n",
    "                    verbose=0, validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.945"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = model.predict_classes(x_data, verbose=0)\n",
    "correct_preds = np.sum(y_pred == y_data, axis=0) \n",
    "sum(correct_preds)/200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X+IZed5H/DvM8raG5SZWUleeWWPQSoxdoKwr9mxJkuL1DZqI0yQkoIhoiQxCoiCq/mVQOIKdu+4Nm2xMzszqqERlWkCjkLAlSvSxLFNaycm2rFnzciVI8mogeKpI3uz0e5KFCXW3qd/nDm6554559zz433f877nfD9wmT137pzz3ru755n3fZ/3eUVVQURE5JuZthtARESUhQGKiIi8xABFREReYoAiIiIvMUAREZGXGKCIiMhLjQOUiBwXkW+IyLMi8h0R2TDRMCIi6jdpug5KRATAjar6mogcA/B1ACuqesFEA4mIqJ9+rOkJNIpwrx0eHjt8cPUvERE10jhAAYCI3ADgIoCfBPAZVd3NeM3DAB4GgBtvvPH0e9/7XhOXJiKiwFy8ePFvVPXktNc1HuKbOJnICQBPAXhEVZ/Le93i4qLu7e0Zuy4REYVDRC6q6uK01xnN4lPVKwC+CuA+k+clIqL+MZHFd/Kw5wQR+XEA9wJ4oel5iYio30zMQd0G4HcP56FmAPyhqv6RgfMSEVGPmcji+zaADxhoCxER5fjRj36Eg4MDvP766203pbTjx49jYWEBx44dq/XzRrL4iIjIroODA8zOzuL2229HtPzUb6qKy5cv4+DgAHfccUetc7DUERFRAF5//XXccsstQQQnABAR3HLLLY16fAxQRESBCCU4xZq2lwGKiIi8xABFRESlPPTQQ7j11ltx5513OrkeAxQRUQeliwSZKBr0kY98BF/84hebn6gkBigioo4ZDoG1tXFQUo2Oh8Nm57377rtx8803N21eaQxQREQdogpcuQJsb4+D1NpadHzlipmelCtcB0VE1CEiwPnz0Z+3t6MHAKysRM+HlAjIHhQRUcckg1QstOAEMEAREXVOPKyXlJyTCgUDFBFRhyTnnFZWgNEo+pqck6rrwQcfxJkzZ/Diiy9iYWEBTzzxhLmGZ+AcFBFRh4gAJ05MzjnFw30nTjQb5nvyySfNNLIkBigioo4ZDqOeUhyM4iDFOSgiImpdOhiFFpwABigiIvIUAxQREXmJAYqIiLzEAEVERF5igCIiospeeOEFnDlzBm9961vx6U9/2so1mGZORESV3XzzzdjZ2cEXvvAFa9dgD4qoIRv77hA1MjcX5ZWnH3Nzxi5x66234oMf/CCOHTtm7JxpDFBEDdjad6dTHNwsKeXVV6s97ykGKKKaurTvjlUduVmSewxQRDXF5WPiQpwzM+MCnSGWlSGa5jOf+QwGgwEGgwG+//3vW78eAxRRA13Zd4eojI9+9KPY39/H/v4+3vGOd1i/HgMUUQNd2XeHqKqXX34ZCwsL2NzcxCc+8QksLCzg2rVrRq/BAEVUk819d4gamZ2t9nwNp06dwsHBAa5du4YrV67g4OAAc4YTX7gOiqgmm/vudMrsbHZChMGbJaUY7sm0hQHKgeS+LFnHFK6u7LtjVUduluQeh/gs4zqZ7uvCvjsUBg1s3LhpexmgLOI6GSIy5fjx47h8+XIwQUpVcfnyZRw/frz2ORoP8YnIuwD8HoBTAEYAHlfV7abn7YLknMT2dvQAuE6GiKpbWFjAwcEBLl261HZTSjt+/DgWFhZq/7w0jcYichuA21T1WyIyC+AigF9Q1b/M+5nFxUXd29trdN2QqEaLOGOjUfPgxHktIgqViFxU1cVpr2s8xKeqf62q3zr886sAngfwzqbn7Qob62R8mtcKrVBqaO0l6jOjc1AicjuADwDYzfjewyKyJyJ7IXVRm7CxTsaneS0bgdJmAPEpsBPRdMYClIj8BIDPA1hV1SN5par6uKouquriyZMnTV3Wa3nrZFZW6q+T8aX+m41AaTOA+BTYiagkVW38AHAMwJ8CWC/z+tOnT2ufjEbFx3XPGd1Wo4eJc9Zpw8rKZDtWVuq1JXmu+BzpYxvtXV6ePHcbnyM5MDs7+RcfP2Zn225ZLwHY0xKxonEPSkQEwBMAnlfVzabn6yLT62R8qf9mslCqi55hVnuTOOTXYdzyI0gmhvj+IYBfBvBPRWT/8PEhA+elDD7VfzMdKG1XBs9q784OsLrKIT8iH5nI4vu6qoqqvk9VB4ePPzbRODrKxrxWHbYSQGz1DLPau7wcfW9nx91cHrMIicpjLb4A+VD/zXSh1HQAOX9+fAw0f39Z7d3air63szN+nc3PcTiMemfxNeL3fOIEhxWJsjBABcqH+m8mA6WLyuDp9mZZW7MTpJJZhMBkAF5Z4UJrokxlMilMP/qWxUfl2ch4zLuOi6zBpOvXzWU9UkXM4vMKSmbxsQdFXnHVM7TdY8sazltfB+bnJ1/HmoyOcMuPIDFAUaH00JNvQ1FN2mdrLq9oOG8wmHytrSFFoi7gdhuUy/fSQCbaZ6PHlremazAA9vfbXx5AFAoGKMqU7AX4WBrI9/Zlrem6//72lwcQhYRDfJTJ972sfG9f1pquq1eBzU1uD09UVuP9oOro235QIVMLe1mZ5GP7itZ0+RJAidrkbD8o6i5fav4lJa89GkVlipLabh/gT7UPotBxiI8y2a7sUEcydXs4BJ5+Oko6WFoC/uIvgNOn221fuq1tV/sgCh0DFGVyUdmhimRSRNxD2t+Pvt51VxQ89/ejTLn5eT8CgQ/VPohCxjkoKuTTOqhkry7LykqUhDDDgWsir3EOiozwqRcwbT+n8+cZnKiEubnoH1P6MTcX1jV6gP+dKRhZSRtJPiRINMYbm30uNi/kBolGcA6KgpAc3kvu4wSMj31JkGiENzaiNzFAURDSSRsbG+PAdNNNwLlz49cEG5yIaAKTJCgoySSN+J9u8jj44FT0BoIfv/SEi8+Yf4+FmCRBnZT8fx9Pz2R9jyhYnId8EwMUUYh4E6tvdrba866vwXnIN3EOisgns7PZN6L0jc3FTWxuLr8tIW8A6KLtIX8+HmGAIvKJTzc2/iZPLeMQHxGOzltzHpuofQxQZFUIN37fdw4m6isGKLImhBu/7zvzEvqXEOIiiSMQnIMiK5I3fuDopn2+rFnyfWfeXGWTKbqgb3NhPs1DtowLdcmarOrjWTd+Hyqmm9iZ14f3YVSVLD6bGX9c9No5XKhLrcuqPp4OTj4MA5rYOdiH92HctWvRG0k/sgJO33o55AQDFFkz7cbvw/xPeufg0Sj6mmxTmXO0/T6IuohzUGSFKrC6GlUcj4f1VlePVhxve/7HxM7BItEuvoPB5PvwaXffYOQNFVIvMUCRFRsbwO5uVHE8Ocy3tDR5448DQnKeynVywnA4OV8Ut6lsG1SBq1fHW9DH9veBu+/uwFyUS1WCUxcTQmgCh/jIuHjIa3d3/NzaWtSbWlqKtsZIvrbp/I8JTXYOFom2mj91avL5wSB6HxsbzdtHKDcXRp1iJECJyGdF5Ici8pyJ85F/qiy4jXsgKytRUJqZGc/xbG1Nbo/RdP7HB/H7ePnlyef394HHHgNeeSWc91Ib1+6QBaZ6UP8FwH2GzkWeqZOhViaDL2/+Z2UlrI0H4/cxGBz93l13TQblykJZpFol44+oJCMBSlX/DMDfmjgX+aVuhlrZobvhcDJwxUFqWnq2TyWUknNQ8S6/sZ/5mYYnZ/o2VRHKLzQlOZuDEpGHRWRPRPYuXbrk6rLUULJXs709OVwXB5Z0cBiNqg3dVZ3/8W3NUdyDSgcnYHIezmu+3Ng4VNhMx36hcRagVPVxVV1U1cWTJ0+6uiwZUDRclxUs1tej3oSNoTtf1hylr3P2bPQ1TqsfjaKAtbsbyHyaLze2UIcKfQnwHcM0c5oqb7huc7O43t7mZv3U7Tw+rJ0aDqP3nexBrq8Dzz472Y547imk+TSqyZcA3zWqauQB4HYAz5V57enTp5XCMBqprqxEv8aurETHy8vj4+vXx8fxI36d7XYlr2n7esnrpj+P5PH160df30h2fyJ6mOTqOm2Ync1+X7Oz5q7hy+fnSzumALCnJWKFqTTzJwE8A+A9InIgIr9m4rzUvnSmXbymZ3k5v2dguyfTxtqp+NxxD255OXtObib1P6rx5+DbnEyIQ1ns3QTLVBbfg6p6m6oeU9UFVX3CxHnJD3GmXTz/s7MTPX/27LicUdLqqr1g0cbaqfQ8WxZrQdm3ORne7P3m2y80DXEOikrZ2IiC0+ZmdLy9PRmYHnkkei4OWLu7wDPPmL9pm6idV0UyKQMY1xRMB+W1Nc/3j5qmT/tLdZnvySQVMUDRVOmb9ObmZO08IKzaeVXkJWUA0TDf1tbknlfBBqmO3dicY4C3ghsWUinJobW0wWCyUGp84w7yRp1DdXJ+Kfke48/mxInA938qI8TNA0Nsc8dxw0IyKi6ImnT9ejTUlq7inQ5Orqo+2LpOVlJGUtnqF9aEmLjgUsfmZfqk8wHKp5I4IYvX+iStrwO//dtHa9AlEwpcVX2wdZ28pIydncnrtdpbLEpcaBK0sgJfHp9v9r4lmlBpnQ5QvpXECVX6Jh33nLa3gcXFcdWIdEbdaOSm6kNyjsz0dbpS0PZNVbLtil5r8mbPHiDlKbNYyvTD9ELd9GLI0Wj6gkpXCzvbkvWZNHHu3OTnFn+e99yT/fy5c5PHthfyVrlOnc/G9OdZqOrC0qLFmU0WbHKRMFmCkgt1g0+SyCo7E09Ynzt3dGLfZUmcthR9Jk16j8nMueRx3vPJ42SCwWhk5/Mvcx1bn41R0yb1m26LXvb/vKvkAiYx9E4vkiSmDe0A0/ck6hrbw11Zx0XVyLMSDGwsqC1zHZufTSPpIa5puCiW+qJMN8v0w+QQX9HQjqvhJd/48r5NDrMWDbFVuY4vn82EqsNzVYf0OMRHnkFfhviA7KEdYHJiP11puw89KRfDatOYGFIrc44q1/Hls3lTlYunx0+zvg/kDwPOzpZPaDBxjjI4xNc7ZYf4Ot2DypvYjyfwu8q3XkKTBIOqvaNp18n7bNJVyJ0y2YMKkYtq4+QVlOxBBR2gyty8nGZftSw9rLm83I3sRVMBN3mewSAKSsnjs2fttD9T3k256BHfsLsWoKh3ygaooGvx1SkcGvKwnmp+tlxyiCu5/fjGxuRnMu2cPor/XpPZmHWGaEWA+flxaab19ag6xte+Fh3fc4/Dz6NsooNmDHGx7hv1RZkoZvrhYh1U1xQNV07bVDDuWYU65Nm0B5V+3RtveDAEWqXHRNQxcLlhYduK0py7QLVcOn1yE72dnaPJIF6mWB9KXz8+Trazzv5PWdVEfv3Xo55UkrGkmbyqCGVLBsWNVG23FA+rO5APykQx0w9u+V5dVi8inkeJv5/eej2rZ9l6zyHDtJ5d3Z5f0RzlYDD5OcTzdcmfraXqvJKvPSnOc2WrmtDBBJBM6EOSRN+MRkf/naeH9KYFn/Q52g5OZbP06g7j5gX25PmXliaDVKOhTxsBqo0bGgNUtqqfCz/HTAxQHVN0o83qCeTd5H3sQdluVzoonz07OTcXB/elJQNZj7YCVJ0bWpnf3utkE5q6dogYoIxggOqQol5GVnBK/kxW0VYfi+fa6tmVWfeU1QOt/Zk0udmbDlBlzmUjUE4LemX5GOQYoIwoG6A6kSTRdVnp9JubR/dhSv9MchO9JttGqBYfN6Vqp15ffN6sBIv19fH5RaJNFpOcVRqpUoMvFKZqBRbtc0X9UCaKmX6wB1VP1lBd1d5Q1bkc26nptnt2ZdpvdIixzpCZqZ5Lmo0eVJnei6n34WPvgz0oI9CHhbp9k6weXnWBcvocecdJquPUdOBoPUPV5r/4N3kvZQyHk+2Mzx8fp3tZyfcI1OhJZaWGh9o7Mt1V7oKqi6S5qLqRThSLDVX6Bl/1ht/058tew8WeWi7eSx7re0Q13b8pVrVIa5lxWxuFWsus82p6HgbPoJUtFssA1ZIgNs47pOpZ9W8LWgmQdS5Q5f9rmWrkNiqWBx6g2vxlqS96sWFhqJJDZz5WdUiK25ZkY8PBtiWH/JI3pPg4SNeuZc9+JANPmddUVTR8VWVoq2jYzJKsyiNra/790tgXDFAtSGbQxaWJfNynKj0/U7XMUGiGQ+DMGWB1dXyfXl2Nnmt8g8oqHdRVeUGvauCzETwLhPSLY18wSaIlpip022Q7gcEnqsArrwC7u9EjtrMTfV1aKjdtk6toDip95/Ptg502h2Z6A8OWJP99b2+P/2/69otjn3AOqiWukg9M6MOY/HAYBShgHJRijzwy/nuqPU9YZT7F5txLnTmnMn/ZHepe9GHOtW2cg/KY66Gz9Pmqnr9qtfiq12vavqbioZ10YEpzNtxjc+7F5eLXACui92XONRQc4muBy6Ez19mCVa/XtH11e3fptVGbm9FzWUHqsceiBxBV75ift/wbdQeGywAEVwnC+Jo4aow9qJYMh5P/4NOliUxwPelb9XpN21c34yrv5y5cmHzd+99/9Gf394GrV/kbdRc1KQdGlpQpNzHtAeA+AC8CeAnAb017PUsdueO6gnnV69VtX90SSUU/9/a3T7bjkUdU3/a2yeeSe3BV4kvh0/z8uvJtLvOzVa/lkT7s0N02uKpmDuAGAP8bwD8A8BYAzwL46aKfYYByy/UeUFWvV7d9ZSqVx6+b9nPx1iXLy9HPZ+2v5SLAW1claJQNTnlBNtAA5bsuBNCyAcrEEN9dAF5S1b9S1b8H8AcAHjBwXjJAHU/6Vr1ek/Yl5+5i8/OTlcrj8yeH/bJ+7v77o6Gcra0og2trK0otf/vbJ1/nZA7KJlMJGMlw05U5swD0bSGxiQD1TgDfSxwfHD43QUQeFpE9Edm7dOmSgcvSNPE/XpfZglWu17R9WcHt6aenz2ll/dzVq1GiRDLwLC0BP/jBZNuCn4Nyufi1hUoQXabaw4XEZbpZRQ8AHwbwnxPHvwzgsaKf4RCfO7a3y2h6vbrtK5pLSu80nLer8LS5q8afnY15pwbnrDw0xCE67/i6K3ZVKDnE13ihroicATBU1Z87PP7YYeD7d3k/w4W6bqm6XWhb9Xp125eXoj4/D3z84+PXpRdaVkltb/TZua4UXnDOWun8BtofvyxZ23DaqamYavgLiV0u1P0mgHeLyB0i8hYAvwTgaQPnJUOqLrR1fb1a7Zubw3BDsLUtkJloAajMCM5/dg5Xr06+ND1cWCXF3/VnZ0N6aGg0mhwaGo1yfrDhEF3T2oZZQ8J953pOuXVlulnTHgA+BOC7iLL5Hp32eg7xUWMFw09lhu/abqPrcxZlPOYNWzYZ4hyNJjMhl5ePHhf9fdgYmg49+8327tMuwVWaeZ0HAxQ1VnCjdjnnVreNteemGgS969cnX55Mp48/s+Sj6c0wHaTKBicbN2LXc7G2dOV9MED1kS+LQV0ouFF785ty1QBVpmdlsAcVP97//ihYxQFlaSm64VWZkM/7zNNr3IDywc1UMkCXeh6q4fcEVRmg+snGkJKvbPROTAf4KpUYyv491Whj+oac7knFvZqsIbgyi6jzfqs/e7ZeDyrZ7qqBrcxn0DTgUXMMUH3EANXsM3D1+bXw9xQHkXjOqagJyeA07aZeNt2/6hyUjYDiuqIK5WOA6qM+BaimvZOqP2+jPTnXsTWEkwxOcbDK6uFUnYPKCyZnz0bDhcmAlxxCzGNjSI49KL+UDVDcboOaqbMBnglNNtcDzG/5YOh8NrdHmZmZrNadZ3U1KvVUdkuYot2h4zbHr9/amjzOYno7mvgz5DYaASoTxUw/2IOypI0eVJ1r2kzmKNueJr2vqtct+RjNzjqZzE/2ZuLLF81BpX8263w2eicme5JdyX7rCrAH1UOzs/m9GZ+Y2sgur/cWCtWJQwFw/vCp7e3xb/jJnoQJ8XluuimqN7i0NO7ZAMDubvS9rOuln7PZOzG5SHo4jNqaXpzNnpPfGpc6qoOljjqkTjkcUyWAqtxd0kOOTX4WMBMcc96ranYpm+QN1pS4CU1KEbnetZnCV7bUEXtQ5Je5OfNzV01/Cctqj6WeW3xzT1pbiyqtr6+bv+mb6KWwd0K2cMt3smdurvrP2Bqym5uL7pjxoyxbw6MZ500Olw0G0XODQXR8+rTbbRXS15h2nFb2I656XuoXBijKlr6hx4900Cm6gfs0P1TUlqLUhTq9uTIpERnnTWavXbw43n8KiL6anovKM21TPFOb5vVt8z2qjgHKN2UDg+1rlU1kqHMD9y1pwyNxpfWZmaOp4C6Ck2rxpnijkZlN86Zdhz0pAsA0c++4TBWvmxKtWn5hqs33WaUNpj9Xy39PbS4snXZtU23j4tn+gqsNC+tgFl8BG5vc1blWEa2QTpbXZpPvc1pbprW3zudqcYFysjeRTt12NcynOZmEZb9v6jrUTS43LCSqruFmeK27ds3cnFVKXiWFlZV6lRTS0vE467hoU7xp36/Sjl5tvkfVlelmmX5wiK9AKEN8ZV5XtTJE3QoTTYYkPd6KxEZNvmkVFabVwUvX86tb7aJrW2BQNWAlCWqkSVWKur8Cm6owkcVmXUBLTG83rzpOTACODh3GI6FFdfDS9fzq1skzXW+PuolzUL5xWXy1ybVszJXVPWfRz9kuWhuY5PxWLGteKz1tV+YYKH5NXnuq/gyFj3NQobI4t2H0Wj7NIRW1xXVwcrlMIEfRHFOypxLLSrqoeryxUW9Nk8leYtH7pjAxQFE9LgMpUHyzd92WPFXWj1kybfFrfJzUNDEhOXTY1pomLvrtJgYoCo9PFSqSWm7XtEAxGk3OOY1G0dfk6+uIe2XLy9G5Zmair8vLfiwuZk8qXEySIDNMzJ3lJWZ0RfpObXgIMjl8l7ddh63EhI2N/Odt92LKvG8KE5MkyAyfkiaqmBZYqwTeuitVDdMpi19NJyaoRrvw7uwc/d7ycrTXlIsgMe19kz+YJEFUxrQ5o5bnlKoqM8dkOn3dBzbm1qh9DFAUphaz5HyVnHsxOcc0jUi0A+/y8uTzy8v5O/Oa1Nb7Jvs4B0X+qjIn5UOPpoU5tOTwnAgwP9/O4tdz56JhvqznbeOi3+5igCJ/mZjjcSkv4cFSu7O2Wr96NQpSLne3jXswOztHi9u62l2Xu/p2E4f4yAyfFu76xvBnEy/zilOrV1cnh7muXi2eczLNdnHbKu0oOqbwMIuPwlGmbk5VJrP4HEj2moCj2XNtplazbBGVVTaLj0N81G/TgoxHdfyyir2mtTmsxR4MmcYhPuqGHgwlJofO4ooN6bVHzFprD2sBmscAReHwqShsS7KKvS4vA9evM7W6TawFaEejACUiHxaR74jISESmjicSNeJLUdgWqQJnzhx9bm1tnGLO1Gq3WAvQnqZzUM8B+BcAfsdAW/zm2WQ59U9cUmh3NzpeXo6ee+yx8fH585Plfsi+tmoB9iEpxUgWn4h8FcBvqGqp1Lwgs/hc1IUjmmI4BF55Jfpzcv5paQl45pnwb1Ah33Rd1gLMWgO3thb1nkMYVvSuFp+IPCwieyKyd+nSJVeXJeqU4TAqvrq1Nfl8HJziUc9Y+thnIc/juKwF2KchxakBSkS+IiLPZTweqHIhVX1cVRdVdfHkyZP1W0xEmTfDc+ei+al44W48JHjmjP83+ZBvuq5rAWZlc8bX7lr1jKlzUKp6r4uGEHnLo/nH9M0wWVZoMAD298dzVMB4GHBpye/hspD3dGqjFmB8jfhzAvz/nOrgHFRZnIPqL8/+7vPmH+bno+fTa6Nc7snUVMh7OrmcP0v+ohILIZjHys5BQVVrPwD8IoADAH8H4AcA/rTMz50+fVqDMzubleAcPU/+a/L3l53cHj1sXrfAaJR9PBodvVT6tU2vmffVxPlXVibbvrJi7vxdkfyc4s8nfew7AHtaIlY0SjNX1acAPNXkHMFgKnnY2tp40NJ1s8oKxXNOaaurzXtQca9tfj4qRru5Cayvj4+bZo8VDV0C4fQMXOjT9iKsxUdh8mheyAdxcIqH9+LNA3d2xs/VDVLJBIZ4nutrX4u+xscrK82GtJI33c3NyZtucvsQivRlexEu6aMwBbYVu23xrrZLS+M5p62t6M9LS812tk1mje3vR88lv5qa+xgOo2C0vj6++W5uRj0037MQ29CH4rzcboPCVDVxoUmiQ5PemuMEi/iU8WXTx03PnVWlwlQiQ9EwX0gJADQdt9sgMiWgIUNbv1VnLUSNra2ZCR4hp5qTHRzio35oa8ffDuw0nOzZDAbRc8mvJhekZlVrDyk4pT8DrkBphj0oOqqLCQhttTvUzyshmcAwPw/cc89kFt8995jLHssrGRRCkAq9Pp6PGKDoqBASEGZn84NoE10MzgYks8bir8kbse05KMDvIJXMdASOzp/5XMXDZwxQFBbbASSE4NySZEpz1lcT5w91fQ/nz+xgFh8dZSrzzEYwsZ0V51lZoz5yWTLItJBLNbnk3XYb1EOmeiNzc9H/cv5P74VQ1/e42nKjT4kYDFDUjjjgiEQBKC0ZlDi8Rp5zteVGyHtm1cE5KDrKVgJCnqxrMShRQFzMn/UxEYNzUGRPlf8t6X+HdYvGNdXBLL6Q53RCY/uzDn2bjRjnoKhfTPXurl3L3lgj0OBUZ0ioT3McptmePwt9IXNVDFBkj4tqCYEHEJuSQ0Jlt1Hv2xxHaFwlYviCAYrsSfZGbAioXFAbklXIt7ej9OeiwquqwCuvTAa01dXigEbuuErE8AkDFLUvK9AU1bCz2WtKZg9OyzQMQJUhoY0N4MKFcX29mZloL6lTp7gnkw/yEjFWVvxfyFwXs/iofVmBpq0hu45Vkihb2y7uPX3jG0fP8fLL4x5UF2+CIenLRoUx9qCIXHPUS6syJCQSbXD4yCNHzzMYdPsmGJpQFzLXwQBFbnRg2wljgcRRL63OkFDWc/v7411uiVziEB+50aUsu4CG+6oOCV24cPS5wYBzUNQO9qDITx1LVmhTmSGhOGMvnoNaXo4eQNSDYhYftYE9KPJTW8kKeWWeOk4EuOkmYGkpemxtjb+3uxt9jz0oco2ljshPPmwcnCV8AAAGiUlEQVR7YasNPry3KZePm5g+JjKBpY6IfOVxwkh6VxPuckJt4hAfUR5bVd27lDBCZBEDFFEeBhKiVnGIj/zk8TAYEbnBHhT5ib0Xot5jD4qIiLzEAEVERF5igCI/sHIEEaU0ClAi8ikReUFEvi0iT4nICVMNo57p2DYXRNRc0x7UlwHcqarvA/BdAB9r3iQiIqKGAUpVv6SqbxweXgCw0LxJRETmpatIsfit/0zOQT0E4E/yvikiD4vInojsXbp0yeBliYiKDYeTmzTGmzkOh222iqaZGqBE5Csi8lzG44HEax4F8AaAz+WdR1UfV9VFVV08efKkmdYTEU2hGm0XktxJON5pmNuI+G3qQl1Vvbfo+yLyqwB+HsDPahul0akbbNW9o96LN2kEoqC0vR39ObnTMPmp0XYbInIfgE0A96hq6XE7brdBRK6pAjOJMaPRiMGpLa622/iPAGYBfFlE9kXkPzU8HxGRcfGwXlJyTor81KgWn6r+pKmGEBHZkJxziof14mOAw3w+Y7FYCt/cXP78FYvO9p4IcOLE5JxTPCd14gSDk8+45TuFz+Mt1MkfqpP/VNLH5A63fCciSkgHIwYn/zFAERGRlxigiIjISwxQRETkJQYoCl9etQlWoSAKGtPMKXxMJSfqJPagiIjISwxQRETkJQYoIiLyEgMUERF5iQGKiIi8xABFREReYoAiIiIvMUAREZGXGKCIiMhLDFBEROQlBigiIvISAxQREXmJAYqIiLzEAEVERF5igCLqkrk5QOToY26u7ZYRVcYARdQlr75a7XkijzFAERGRlxigiIjISwxQRETkJQYoIiLyEgMUUZfMzlZ7nshjP9Z2A4jIoGvX2m4BkTHsQRERkZcYoIiIyEuNApSI/FsR+baI7IvIl0TkHaYaRkRE/da0B/UpVX2fqg4A/BGAswbaRERE1CxAqWpyRvZGANqsOURERJHGWXwi8kkAvwLgKoB/UvC6hwE8fHj4dyLyXNNrB+xtAP6m7Ua0qO/vH+BnwPff7/f/njIvEtXiTo+IfAXAqYxvPaqq/y3xuo8BOK6q56ZeVGRPVRfLNLCL+P77/f4BfgZ8/3z/Zd7/1B6Uqt5b8pq/D+C/A5gaoIiIiKZpmsX37sTh/QBeaNYcIiKiSNM5qH8vIu8BMALwfwD8q5I/93jD64aO75/6/hnw/fdbqfc/dQ6KiIioDawkQUREXmKAIiIiL7UWoPpeJklEPiUiLxx+Bk+JyIm22+SSiHxYRL4jIiMR6U26rYjcJyIvishLIvJbbbfHNRH5rIj8sI/rIEXkXSLyP0Xk+cN/+yttt8k1ETkuIt8QkWcPP4ONwte3NQclInNxJQoRWQbw06paNskieCLyzwH8D1V9Q0T+AwCo6m+23CxnROSnECXX/A6A31DVvZabZJ2I3ADguwD+GYADAN8E8KCq/mWrDXNIRO4G8BqA31PVO9tuj0sichuA21T1WyIyC+AigF/o2d+/ALhRVV8TkWMAvg5gRVUvZL2+tR5U38skqeqXVPWNw8MLABbabI9rqvq8qr7YdjscuwvAS6r6V6r69wD+AMADLbfJKVX9MwB/23Y72qCqf62q3zr886sAngfwznZb5ZZGXjs8PHb4yL33tzoHJSKfFJHvAfiX6Heh2YcA/EnbjSDr3gnge4njA/TsBkUREbkdwAcA7LbbEvdE5AYR2QfwQwBfVtXcz8BqgBKRr4jIcxmPBwBAVR9V1XcB+ByAf22zLW2Y9v4PX/MogDcQfQadUub994xkPNerkQMCROQnAHwewGpqJKkXVPX64Q4YCwDuEpHcoV6rW773vUzStPcvIr8K4OcB/Kx2cEFahb//vjgA8K7E8QKA77fUFmrB4bzL5wF8TlX/a9vtaZOqXhGRrwK4D0Bm0kybWXy9LpMkIvcB+E0A96vq/2u7PeTENwG8W0TuEJG3APglAE+33CZy5DBB4AkAz6vqZtvtaYOInIwzlkXkxwHci4J7f5tZfJ9HVHL9zTJJqvp/W2lMC0TkJQBvBXD58KkLPcti/EUAjwE4CeAKgH1V/bl2W2WfiHwIwBaAGwB8VlU/2XKTnBKRJwH8Y0TbTfwAwDlVfaLVRjkiIv8IwJ8D+F+I7nsA8G9U9Y/ba5VbIvI+AL+L6N//DIA/VNWP576+gyNLRETUAawkQUREXmKAIiIiLzFAERGRlxigiIjISwxQRETkJQYoIiLyEgMUERF56f8DVTaL2KDDEKcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_pred = np.squeeze(y_pred)\n",
    "plt.scatter(X_xor[y_pred == 1, 0], X_xor[y_pred == 1, 1], c='b', marker='x', label='1')\n",
    "plt.scatter(X_xor[y_pred == 0, 0], X_xor[y_pred == 0, 1], c='r', marker='s', label='-1')\n",
    "\n",
    "plt.xlim([-3, 3])\n",
    "plt.ylim([-3, 3])\n",
    "plt.legend(loc='best')\n",
    "plt.tight_layout()\n",
    "# plt.savefig('./figures/xor.png', dpi=300)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
